From 8adfdbd6c6ee853da06638eca27a5b4a72f357fe Mon Sep 17 00:00:00 2001 From: medjed02 Date: Tue, 10 Nov 2026 19:16:19 +0300 Subject: [PATCH] Add CacheAllocatorConfig::usePosixShmDirectory() POSIX shm segments always live in /dev/shm, which is lost when it is replaced, e.g. on container recreation. usePosixShmDirectory(dir) keeps normal-page segments as files in a user-provided directory (a tmpfs volume that outlives the container) instead. Huge-page segments keep using hugePageMountDir, and PosixShmSegment now uses its directory argument for normal pages too; ShmManager passes one only for huge pages or with this option. To keep the change small, the directory is not saved with the cache: SharedMemAttach with a different directory throws, and the files in the previous custom directory are left for the user to delete. Cross-host persistence rejects the option, as it does huge pages. --- cachelib/allocator/CacheAllocator.h | 19 +++--- cachelib/allocator/CacheAllocatorConfig.h | 22 +++++++ cachelib/allocator/ReadOnlySharedCacheView.h | 7 ++- .../tests/CacheAllocatorConfigTest.cpp | 62 +++++++++++++++++++ cachelib/persistence/PersistenceManager.h | 2 + cachelib/shm/PosixShmSegment.cpp | 5 +- cachelib/shm/PosixShmSegment.h | 16 ++--- cachelib/shm/Shm.h | 8 +-- cachelib/shm/ShmManager.cpp | 54 ++++++++++------ cachelib/shm/ShmManager.h | 21 +++++-- .../Cache_persistence.md | 4 ++ 11 files changed, 173 insertions(+), 47 deletions(-) diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index b485a5fbab..7f9afe6c2c 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1499,7 +1499,8 @@ class CacheAllocator : public CacheBase { // and the cache was actually attached. static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix, - const std::string& hugePageMountDir = ""); + const std::string& hugePageMountDir = "", + const std::string& posixShmDir = ""); // gives a relative offset to a pointer within the cache. uint64_t getItemPtrAsOffset(const void* ptr); @@ -2949,7 +2950,8 @@ CacheAllocator::CacheAllocator( shmManager_(type != InitMemType::kNone ? std::make_unique(config_.cacheDir, config_.isUsingPosixShm(), - config_.hugePageMountDir) + config_.hugePageMountDir, + config_.posixShmDirectory) : nullptr), deserializer_(type == InitMemType::kMemAttach ? createDeserializer() : nullptr), @@ -7048,12 +7050,13 @@ template bool CacheAllocator::cleanupStrayShmSegments( const std::string& cacheDir, bool posix, - const std::string& hugePageMountDir) { + const std::string& hugePageMountDir, + const std::string& posixShmDir) { if (util::getStatIfExists(cacheDir, nullptr) && util::isDir(cacheDir)) { try { // cache dir exists. clean up only if there are no other processes // attached. if another process was attached, the following would fail. - ShmManager::cleanup(cacheDir, posix, hugePageMountDir); + ShmManager::cleanup(cacheDir, posix, hugePageMountDir, posixShmDir); } catch (const std::exception& e) { XLOGF(ERR, "Error cleaning up {}. Exception: ", cacheDir, e.what()); return false; @@ -7063,13 +7066,13 @@ bool CacheAllocator::cleanupStrayShmSegments( // Any other concurrent process can not be attached to the segments or // even if it does, we want to mark it for destruction. ShmManager::removeByName(cacheDir, detail::kShmInfoName, posix, - hugePageMountDir); + hugePageMountDir, posixShmDir); ShmManager::removeByName(cacheDir, detail::kShmCacheName, posix, - hugePageMountDir); + hugePageMountDir, posixShmDir); ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix, - hugePageMountDir); + hugePageMountDir, posixShmDir); ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName, - posix, hugePageMountDir); + posix, hugePageMountDir, posixShmDir); } return true; } diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index ea52537a26..fb0441a5e4 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -239,6 +239,13 @@ class CacheAllocatorConfig { // cachePersistence() CacheAllocatorConfig& usePosixForShm(); + // Like usePosixForShm(), but normal-page segments are files in the given + // directory instead of /dev/shm. Use an existing tmpfs directory. It is not + // saved with the cache: SharedMemAttach with another directory throws. + // @throw std::invalid_argument if the directory is empty or called without + // enabling cachePersistence() + CacheAllocatorConfig& usePosixShmDirectory(std::string directory); + // Requests HugeTLB-backed pages for slab (main cache) memory and access // container hash tables; persistence metadata stays on normal pages. // Shorthand for enableSlabHugePages + enableAccessHugePages with the same @@ -571,6 +578,9 @@ class CacheAllocatorConfig { // if true, uses posix shm; if not, uses sys-v (default) bool usePosixShm{false}; + // Directory for normal-page POSIX shm segments; empty means /dev/shm. + std::string posixShmDirectory; + // Huge-page size backing the slab memory segment; default => normal pages. PageSize slabHugePageSize{}; @@ -1143,6 +1153,17 @@ CacheAllocatorConfig& CacheAllocatorConfig::usePosixForShm() { return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::usePosixShmDirectory( + std::string directory) { + if (directory.empty()) { + throw std::invalid_argument("usePosixShmDirectory() needs a directory"); + } + usePosixForShm(); + posixShmDirectory = std::move(directory); + return *this; +} + template void CacheAllocatorConfig::validateHugePageSize(const PageSize& pageSize) { if (!pageSize.isHugePage()) { @@ -1543,6 +1564,7 @@ std::map CacheAllocatorConfig::serialize() const { configMap["slabHugePageSize"] = pageSizeStr(slabHugePageSize); configMap["accessHugePageSize"] = pageSizeStr(accessHugePageSize); configMap["hugePageMountDir"] = hugePageMountDir; + configMap["posixShmDirectory"] = posixShmDirectory; configMap["defaultAllocSizes"] = ""; // Stringify std::set diff --git a/cachelib/allocator/ReadOnlySharedCacheView.h b/cachelib/allocator/ReadOnlySharedCacheView.h index 4ed4b61308..d1af3b9cf4 100644 --- a/cachelib/allocator/ReadOnlySharedCacheView.h +++ b/cachelib/allocator/ReadOnlySharedCacheView.h @@ -48,17 +48,20 @@ class ReadOnlySharedCacheView { // address chosen by the kernel // @param pageSize page size backing the cache's main segment // @param hugePageMountDir hugetlbfs mount containing a POSIX huge-page cache + // @param posixShmDir the cache's usePosixShmDirectory(), if any explicit ReadOnlySharedCacheView(const std::string& cacheDir, bool usePosixShm, void* addr = nullptr, const PageSize& pageSize = PageSize(), - const std::string& hugePageMountDir = "") + const std::string& hugePageMountDir = "", + const std::string& posixShmDir = "") : shm_(ShmManager::attachShmReadOnly(cacheDir, detail::kShmCacheName, usePosixShm, addr, pageSize, - hugePageMountDir)) {} + hugePageMountDir, + posixShmDir)) {} // returns the absolute address at which the shared memory mapping is mounted. // The caller can add a relative offset obtained from diff --git a/cachelib/allocator/tests/CacheAllocatorConfigTest.cpp b/cachelib/allocator/tests/CacheAllocatorConfigTest.cpp index a8fa70b5ac..1e5d0f08f0 100644 --- a/cachelib/allocator/tests/CacheAllocatorConfigTest.cpp +++ b/cachelib/allocator/tests/CacheAllocatorConfigTest.cpp @@ -14,8 +14,14 @@ * limitations under the License. */ +#include + +#include +#include + #include "cachelib/allocator/CacheAllocatorConfig.h" #include "cachelib/allocator/MemoryTierCacheConfig.h" +#include "cachelib/allocator/ReadOnlySharedCacheView.h" #include "cachelib/allocator/memory/Slab.h" #include "cachelib/allocator/tests/TestBase.h" #include "cachelib/common/EventSink.h" @@ -122,6 +128,62 @@ TEST_F(CacheAllocatorConfigTest, ConflictingHugePageMountDirsThrow) { EXPECT_EQ("/mnt/hugetlbfs-a", config.hugePageMountDir); } +TEST_F(CacheAllocatorConfigTest, PosixShmDirectory) { + AllocatorT::Config config; + // Needs cache persistence, as usePosixForShm() does. + EXPECT_THROW(config.usePosixShmDirectory("/segments"), std::invalid_argument); + config.enableCachePersistence("/cache"); + EXPECT_THROW(config.usePosixShmDirectory(""), std::invalid_argument); + + config.usePosixShmDirectory("/segments"); + EXPECT_TRUE(config.isUsingPosixShm()); + EXPECT_EQ("/segments", config.posixShmDirectory); + EXPECT_EQ("/segments", config.serialize()["posixShmDirectory"]); +} + +TEST_F(CacheAllocatorConfigTest, PosixShmDirectoryWarmRoll) { + folly::test::TemporaryDirectory cacheDir; + folly::test::TemporaryDirectory segmentDir("", "/dev/shm"); + const std::string segments = segmentDir.path().string(); + const auto makeConfig = [&](const std::string& directory) { + AllocatorT::Config config; + config.setCacheSize(64 * Slab::kSize) + .enableCachePersistence(cacheDir.path().string()) + .usePosixShmDirectory(directory); + return config; + }; + { + AllocatorT cache(AllocatorT::SharedMemNew, makeConfig(segments)); + const auto pool = + cache.addPool("default", cache.getCacheMemoryStats().ramCacheSize); + auto handle = cache.allocate(pool, "key", 100); + ASSERT_NE(nullptr, handle); + std::memset(handle->getMemory(), 'v', 100); + cache.insertOrReplace(handle); + handle.reset(); + ASSERT_EQ(AllocatorT::ShutDownStatus::kSuccess, cache.shutDown()); + } + ASSERT_FALSE(std::filesystem::is_empty(segments)); + { + ReadOnlySharedCacheView view(cacheDir.path().string(), true /* posix */, + nullptr, PageSize(), "", segments); + EXPECT_NE(0u, view.getShmMappingAddress()); + } + + { + AllocatorT cache(AllocatorT::SharedMemAttach, makeConfig(segments)); + auto handle = cache.find("key"); + ASSERT_NE(nullptr, handle); + EXPECT_EQ('v', *reinterpret_cast(handle->getMemory())); + handle.reset(); + ASSERT_EQ(AllocatorT::ShutDownStatus::kSuccess, cache.shutDown()); + } + + EXPECT_TRUE(AllocatorT::cleanupStrayShmSegments( + cacheDir.path().string(), true /* posix */, "", segments)); + EXPECT_TRUE(std::filesystem::is_empty(segments)); +} + TEST_F(CacheAllocatorConfigTest, SerializeEvictionPolicyLru) { LruAllocator::Config config; auto serialized = config.serialize(); diff --git a/cachelib/persistence/PersistenceManager.h b/cachelib/persistence/PersistenceManager.h index d71f27c9d7..82e688cd69 100644 --- a/cachelib/persistence/PersistenceManager.h +++ b/cachelib/persistence/PersistenceManager.h @@ -119,6 +119,8 @@ class PersistenceManager { CACHELIB_CHECK_THROW(!config.slabHugePageSize.isHugePage() && !config.accessHugePageSize.isHugePage(), "Huge pages are not supported to persist"); + CACHELIB_CHECK_THROW(config.posixShmDirectory.empty(), + "usePosixShmDirectory() is not supported to persist"); if (config.nvmConfig.has_value()) { const auto& navyConfig = config.nvmConfig->navyConfig; diff --git a/cachelib/shm/PosixShmSegment.cpp b/cachelib/shm/PosixShmSegment.cpp index 0cefe2156c..31a0dd3adb 100644 --- a/cachelib/shm/PosixShmSegment.cpp +++ b/cachelib/shm/PosixShmSegment.cpp @@ -223,11 +223,12 @@ int shmOpenForPageSize(const std::string& name, int flags, const PageSize& pageSize, const std::string& hugePageMountDir) { - if (!pageSize.isHugePage()) { + if (!pageSize.isHugePage() && hugePageMountDir.empty()) { return detail::shmOpenImpl(name.c_str(), flags); } // HugeTLB segments must use a different mount from /dev/shm (which is - // hardcoded by shm_open) + // hardcoded by shm_open). Normal-page segments get a directory only from + // usePosixShmDirectory(). if (hugePageMountDir.empty()) { util::throwSystemError( EINVAL, diff --git a/cachelib/shm/PosixShmSegment.h b/cachelib/shm/PosixShmSegment.h index 71c23a8228..f2e518f11e 100644 --- a/cachelib/shm/PosixShmSegment.h +++ b/cachelib/shm/PosixShmSegment.h @@ -49,8 +49,8 @@ class PosixShmSegment : public ShmBase { // // @param name Name of the segment // @param opts the options for attaching to the segment. - // @param hugePageMountDir hugetlbfs mount backing the segment when it uses - // huge pages; empty for normal (tmpfs) segments. + // @param hugePageMountDir directory of the segment: the hugetlbfs mount for + // huge pages; for normal pages "" means /dev/shm. PosixShmSegment(ShmAttachT, const std::string& name, ShmSegmentOpts opts = {}, @@ -60,8 +60,8 @@ class PosixShmSegment : public ShmBase { // @param name The name of the segment // @param size The size of the segment. This will be rounded up to the // nearest page size. - // @param hugePageMountDir hugetlbfs mount to back the segment on when it - // uses huge pages; empty for normal (tmpfs) segments. + // @param hugePageMountDir directory of the segment: the hugetlbfs mount for + // huge pages; for normal pages "" means /dev/shm. PosixShmSegment(ShmNewT, const std::string& name, size_t size, @@ -98,8 +98,8 @@ class PosixShmSegment : public ShmBase { // useful for removing without attaching // @param name Name of the segment - // @param hugePageMountDir hugetlbfs mount backing the segment if it uses - // huge pages; empty for normal (tmpfs) segments. + // @param hugePageMountDir directory to remove the segment from in addition + // to /dev/shm, if not empty. // @return true if the segment existed. false otherwise static bool removeByName(const std::string& name, const std::string& hugePageMountDir = ""); @@ -126,8 +126,8 @@ class PosixShmSegment : public ShmBase { void memBind(void* addr) const; - // hugetlbfs mount backing this segment when it uses huge pages ("" for - // normal/tmpfs segments). Needed to locate the file for removal. + // Directory of the segment file ("" for /dev/shm). Needed to locate the + // file for removal. const std::string hugePageMountDir_; // file descriptor associated with the shm. This has FD_CLOEXEC set diff --git a/cachelib/shm/Shm.h b/cachelib/shm/Shm.h index 75e18c8a82..17135106f4 100644 --- a/cachelib/shm/Shm.h +++ b/cachelib/shm/Shm.h @@ -47,8 +47,8 @@ class ShmSegment { // @param name name of the segment // @param size size of the segment. // @param opts the options for the segment. - // @param hugePageMountDir hugetlbfs mount for a huge-page POSIX segment; - // ignored for SysV and normal-page segments. + // @param hugePageMountDir directory of a POSIX segment: the hugetlbfs mount + // for huge pages; for normal pages "" means /dev/shm. Unused by SysV. ShmSegment(ShmNewT, std::string name, size_t size, @@ -67,8 +67,8 @@ class ShmSegment { // attach to an existing segment with the given key // @param name name of the segment // @param opts the options for the segment. - // @param hugePageMountDir hugetlbfs mount for a huge-page POSIX segment; - // ignored for SysV and normal-page segments. + // @param hugePageMountDir directory of a POSIX segment: the hugetlbfs mount + // for huge pages; for normal pages "" means /dev/shm. Unused by SysV. ShmSegment(ShmAttachT, std::string name, bool usePosix, diff --git a/cachelib/shm/ShmManager.cpp b/cachelib/shm/ShmManager.cpp index 8c982ea5b2..d8d199635b 100644 --- a/cachelib/shm/ShmManager.cpp +++ b/cachelib/shm/ShmManager.cpp @@ -41,9 +41,16 @@ inline std::string pathName(const std::string& dir, const std::string& file) { bool removeSegByName(bool posix, const std::string& uniqueName, - const std::string& hugePageMountDir) { - return posix ? PosixShmSegment::removeByName(uniqueName, hugePageMountDir) - : SysVShmSegment::removeByName(uniqueName); + const std::string& hugePageMountDir, + const std::string& posixShmDir) { + if (!posix) { + return SysVShmSegment::removeByName(uniqueName); + } + bool removed = PosixShmSegment::removeByName(uniqueName, hugePageMountDir); + if (!posixShmDir.empty()) { + removed |= PosixShmSegment::removeByName(uniqueName, posixShmDir); + } + return removed; } std::string uniqueIdForNameAndDir(const std::string& name, @@ -54,11 +61,13 @@ std::string uniqueIdForNameAndDir(const std::string& name, ShmManager::ShmManager(const std::string& dir, bool usePosix, - const std::string& hugePageMountDir) + const std::string& hugePageMountDir, + const std::string& posixShmDir) : controlDir_(dir), dirHash_( std::to_string(XXH3_64bits(controlDir_.data(), controlDir_.size()))), hugePageMountDir_(hugePageMountDir), + posixShmDir_(posixShmDir), usePosix_(usePosix) { // check that the directory exists. If it does not exist, create // one. If it exists, extract the name to key mapping from the metadata @@ -171,7 +180,8 @@ bool ShmManager::initFromFile() { const auto removeFromMount = [&](const std::string& mount) { for (const auto& entry : nameToKey_) { try { - removeSegByName(usePosix_, uniqueIdForName(entry.first), mount); + removeSegByName(usePosix_, uniqueIdForName(entry.first), mount, + posixShmDir_); } catch (...) { if (!firstError) { firstError = std::current_exception(); @@ -256,8 +266,10 @@ typename ShmManager::ShutDownRes ShmManager::shutDown() { void ShmManager::removeByName(const std::string& dir, const std::string& name, bool posix, - const std::string& hugePageMountDir) { - removeSegByName(posix, uniqueIdForNameAndDir(name, dir), hugePageMountDir); + const std::string& hugePageMountDir, + const std::string& posixShmDir) { + removeSegByName(posix, uniqueIdForNameAndDir(name, dir), hugePageMountDir, + posixShmDir); } bool ShmManager::segmentExists(const std::string& cacheDir, @@ -277,11 +289,12 @@ std::unique_ptr ShmManager::attachShmReadOnly( bool posix, void* addr, const PageSize& pageSize, - const std::string& hugePageMountDir) { + const std::string& hugePageMountDir, + const std::string& posixShmDir) { ShmSegmentOpts opts{pageSize, true /* read only */}; - auto shm = - std::make_unique(ShmAttach, uniqueIdForNameAndDir(name, dir), - posix, opts, hugePageMountDir); + auto shm = std::make_unique( + ShmAttach, uniqueIdForNameAndDir(name, dir), posix, opts, + pageSize.isHugePage() ? hugePageMountDir : posixShmDir); if (!shm->mapAddress(addr)) { throw std::invalid_argument(fmt::format( "Error mapping shm {} under {}, addr: {}", name, dir, addr)); @@ -291,15 +304,17 @@ std::unique_ptr ShmManager::attachShmReadOnly( void ShmManager::cleanup(const std::string& dir, bool posix, - const std::string& hugePageMountDir) { + const std::string& hugePageMountDir, + const std::string& posixShmDir) { // instantiate a shm manager and destroy it to clear all the segments // associated with the directory. - ShmManager s(dir, posix, hugePageMountDir); + ShmManager s(dir, posix, hugePageMountDir, posixShmDir); } void ShmManager::removeAllSegments() { for (const auto& kv : nameToKey_) { - removeSegByName(usePosix_, uniqueIdForName(kv.first), hugePageMountDir_); + removeSegByName(usePosix_, uniqueIdForName(kv.first), hugePageMountDir_, + posixShmDir_); } nameToKey_.clear(); } @@ -310,7 +325,8 @@ void ShmManager::removeUnAttachedSegments() { const auto name = it->first; // check if the segment is attached. if (!segments_.contains(name)) { // not attached - removeSegByName(usePosix_, uniqueIdForName(name), hugePageMountDir_); + removeSegByName(usePosix_, uniqueIdForName(name), hugePageMountDir_, + posixShmDir_); it = nameToKey_.erase(it); } else { ++it; @@ -334,7 +350,7 @@ ShmAddr ShmManager::createShm(const std::string& shmName, try { newSeg = std::make_unique(ShmNew, uniqueIdForName(shmName), size, - usePosix_, opts, hugePageMountDir_); + usePosix_, opts, segmentDir(opts)); } catch (const std::system_error& e) { // if segment already exists by this key and we dont know about // it(EEXIST), its an invalid state. @@ -374,7 +390,7 @@ ShmManager::attachNewShm(const std::string& shmName, // This means the segment exists and we can try to attach it. try { auto seg = std::make_unique(ShmAttach, uniqueIdForName(shmName), - usePosix_, opts, hugePageMountDir_); + usePosix_, opts, segmentDir(opts)); auto [it, _] = segments_.emplace(shmName, std::move(seg)); return it; } catch (const std::system_error& e) { @@ -419,8 +435,8 @@ bool ShmManager::removeShm(const std::string& shmName) { DCHECK(shm.isInvalid()); } catch (const std::invalid_argument&) { // shm by this name is not attached. - if (!removeSegByName(usePosix_, uniqueIdForName(shmName), - hugePageMountDir_)) { + if (!removeSegByName(usePosix_, uniqueIdForName(shmName), hugePageMountDir_, + posixShmDir_)) { DCHECK(!segments_.contains(shmName)); DCHECK(!nameToKey_.contains(shmName)); return false; diff --git a/cachelib/shm/ShmManager.h b/cachelib/shm/ShmManager.h index 9df87782ab..16359f7014 100644 --- a/cachelib/shm/ShmManager.h +++ b/cachelib/shm/ShmManager.h @@ -49,7 +49,8 @@ class ShmManager { public: ShmManager(const std::string& dirName, bool usePosix, - const std::string& hugePageMountDir = ""); + const std::string& hugePageMountDir = "", + const std::string& posixShmDir = ""); ShmManager(const ShmManager&) = delete; ShmManager& operator=(const ShmManager&) = delete; @@ -129,7 +130,8 @@ class ShmManager { static void removeByName(const std::string& cacheDir, const std::string& segName, bool posix, - const std::string& hugePageMountDir = ""); + const std::string& hugePageMountDir = "", + const std::string& posixShmDir = ""); // Useful for checking whether a segment exists by name associated with a // given cacheDir without instanciating. This should be ONLY used in tests. @@ -140,7 +142,8 @@ class ShmManager { // free up and remove all the segments related to the cache directory. static void cleanup(const std::string& cacheDir, bool posix, - const std::string& hugePageMountDir = ""); + const std::string& hugePageMountDir = "", + const std::string& posixShmDir = ""); // expose a read only instance of the shm for use where we want to peek at a // particular segment without owning its lifetime or any guarantees. @@ -153,6 +156,8 @@ class ShmManager { // @param pageSize page size backing the segment // @param hugePageMountDir hugetlbfs mount containing a POSIX huge-page // segment + // @param posixShmDir directory containing a POSIX normal-page segment; + // empty means /dev/shm // // @return a unique_ptr to the shm if one is present with the name. static std::unique_ptr attachShmReadOnly( @@ -161,7 +166,8 @@ class ShmManager { bool posix, void* addr = nullptr, const PageSize& pageSize = PageSize(), - const std::string& hugePageMountDir = ""); + const std::string& hugePageMountDir = "", + const std::string& posixShmDir = ""); private: enum class ShmVal : int8_t { SHM_SYS_V = 1, SHM_POSIX, SHM_INVALID }; @@ -234,6 +240,13 @@ class ShmManager { // removal when no ShmSegmentOpts are available. Empty if none use huge pages. const std::string hugePageMountDir_{}; + // Directory of normal-page POSIX segments; empty means /dev/shm. + const std::string posixShmDir_{}; + + const std::string& segmentDir(const ShmSegmentOpts& opts) const { + return opts.pageSize.isHugePage() ? hugePageMountDir_ : posixShmDir_; + } + // file handle for the metadata file. It remains open throughout the lifetime // of the object. std::ofstream metadataStream_; diff --git a/website/docs/Cache_Library_User_Guides/Cache_persistence.md b/website/docs/Cache_Library_User_Guides/Cache_persistence.md index cd1e0f03c4..8e909fed67 100644 --- a/website/docs/Cache_Library_User_Guides/Cache_persistence.md +++ b/website/docs/Cache_Library_User_Guides/Cache_persistence.md @@ -131,6 +131,10 @@ Changing the backing type does not cold-roll: `ShmManager` rejects the attach wi Huge pages are not supported with cross-host cache persistence. They're also not supported with memory monitoring when the *slab* huge page size is bigger than the slab size - for example, you can't punch a slab-sized hole in a 1GB huge page. (The access huge-page size is exempt: hash tables are never `madvise`d away, so access-only huge pages work with memory monitoring regardless of size.) +## Keep POSIX segments in a directory + +By default POSIX shared memory segments live in `/dev/shm`. To keep them as files in another directory, for example a tmpfs volume that outlives a container restart, call `usePosixShmDirectory(dir)` instead of `usePosixForShm()`. The directory must exist and should be on tmpfs; a disk filesystem would write the cache back to disk. The directory is not saved with the cache: if it differs from the previous run, `SharedMemAttach` throws as if there were no saved cache, and segment files in a previous custom directory are left for you to delete. Huge-page segments still use their `hugetlbfs` mount. Cross-host persistence does not support this option yet. + ## Drop persistent cache Sometimes you would like your cache to be not persistent when you restart your process. There are two ways to accomplish this: