diff --git a/indra/alscript/lsl/allslservice.cpp b/indra/alscript/lsl/allslservice.cpp index 6d861513f1..cdfecfbc73 100644 --- a/indra/alscript/lsl/allslservice.cpp +++ b/indra/alscript/lsl/allslservice.cpp @@ -36,6 +36,7 @@ #include "allsltraits.h" #include "alscriptlexicon.h" +#include "llcoros.h" #include "llfile.h" #include "Luau/LSLBuiltins.h" @@ -47,11 +48,13 @@ #include #include -#include +#include #include #include #include +#include LLCOROS_MUTEX_HEADER + namespace { std::atomic sBuiltinsLoaded{ false }; @@ -1110,33 +1113,38 @@ namespace } // The builtins of a kind: its functions, or its constants; none of any - // other kind. - const std::vector& builtinNames(Tailslide::LSLSymbolTable* builtins, Tailslide::LSLSymbolType wanted) - { - static std::mutex lock; - static Tailslide::LSLSymbolTable* listed = nullptr; - static size_t count = 0; - static std::vector functions; - static std::vector variables; - static const std::vector none; - const std::lock_guard guard(lock); + // other kind. Asked on any thread, and in a coroutine: each list is + // handed out shared, and made anew rather than changed where the table + // changes, so that one being read outlives its place here. + typedef std::shared_ptr> names_t; + names_t builtinNames(Tailslide::LSLSymbolTable* builtins, Tailslide::LSLSymbolType wanted) + { + static LLCoros::Mutex lock; + static Tailslide::LSLSymbolTable* listed = nullptr; + static size_t count = 0; + static names_t functions = std::make_shared>(); + static names_t variables = functions; + static const names_t none = functions; + const LLCoros::LockType guard(lock); if (builtins && (builtins != listed || builtins->getMap().size() != count)) { - functions.clear(); - variables.clear(); + auto made_functions = std::make_shared>(); + auto made_variables = std::make_shared>(); for (auto& [name, symbol] : builtins->getMap()) { if (symbol->getSymbolType() == Tailslide::SYM_FUNCTION) { - functions.emplace_back(symbol->getName()); + made_functions->emplace_back(symbol->getName()); } else if (symbol->getSymbolType() == Tailslide::SYM_VARIABLE) { - variables.emplace_back(symbol->getName()); + made_variables->emplace_back(symbol->getName()); } } - listed = builtins; - count = builtins->getMap().size(); + functions = std::move(made_functions); + variables = std::move(made_variables); + listed = builtins; + count = builtins->getMap().size(); } return wanted == Tailslide::SYM_FUNCTION ? functions : wanted == Tailslide::SYM_VARIABLE ? variables : none; } @@ -1661,7 +1669,7 @@ ALScriptProblems ALLSLService::check(std::string_view source, bool mono) // the handler it is in, nor a variable to a label. const Tailslide::LSLSymbolType wanted = kindWantedAt(script, problem.line, problem.column); ALScriptFixes::offerNames(problem, lines, problem.args[0], namesAt(script, problem.line, problem.column, wanted), - builtinNames(parser.context.builtins, wanted)); + *builtinNames(parser.context.builtins, wanted)); } } } diff --git a/indra/alscript/luau/alluauconfigscript.cpp b/indra/alscript/luau/alluauconfigscript.cpp index baf2371572..c618ab1554 100644 --- a/indra/alscript/luau/alluauconfigscript.cpp +++ b/indra/alscript/luau/alluauconfigscript.cpp @@ -30,15 +30,17 @@ #include "Luau/Compiler.h" #include "lua.h" #include "lualib.h" +#include "llcoros.h" #include #include #include #include #include -#include #include +#include LLCOROS_MUTEX_HEADER + LUAU_FASTFLAG(LuauIntegerLibrary) namespace @@ -377,7 +379,7 @@ namespace // than its text: a thread held up elsewhere would otherwise see a good // configuration refused until the text changed. Kept a while all the // same, not run again at every ask, so that one that never ends costs - // its time only so often. + // its time only so often. Asked on any thread, and in a coroutine. struct Ran { std::string source; @@ -387,7 +389,7 @@ namespace }; constexpr size_t RAN_KEPT = 32; constexpr std::chrono::seconds LATE_KEPT = std::chrono::seconds(10); - std::mutex sRanMutex; + LLCoros::Mutex sRanMutex; std::vector sRan; } @@ -395,7 +397,7 @@ namespace bool ALLuauConfigScript::asLuaurc(const std::string& source, std::string& json, std::string& error) { { - const std::lock_guard lock(sRanMutex); + const LLCoros::LockType lock(sRanMutex); const auto found = std::find_if(sRan.begin(), sRan.end(), [&source](const Ran& ran) { return ran.source == source; }); if (found != sRan.end() && std::chrono::steady_clock::now() < found->until) { @@ -423,7 +425,7 @@ bool ALLuauConfigScript::asLuaurc(const std::string& source, std::string& json, { ran.until = std::chrono::steady_clock::now() + LATE_KEPT; } - const std::lock_guard lock(sRanMutex); + const LLCoros::LockType lock(sRanMutex); sRan.insert(sRan.begin(), std::move(ran)); if (sRan.size() > RAN_KEPT) { diff --git a/indra/alscript/luau/alluauexports.cpp b/indra/alscript/luau/alluauexports.cpp index 23ddbd06d2..c50eb097f3 100644 --- a/indra/alscript/luau/alluauexports.cpp +++ b/indra/alscript/luau/alluauexports.cpp @@ -29,11 +29,13 @@ #include "Luau/Ast.h" #include "Luau/Parser.h" #include "alscriptlexicon.h" +#include "llcoros.h" #include #include #include -#include + +#include LLCOROS_MUTEX_HEADER namespace { @@ -219,7 +221,8 @@ namespace ALLuauExports namespace { // What each module checked was found to export, by its key and its - // text's length and hash, the latest last. + // text's length and hash, the latest last: kept and asked on any + // thread, and in a coroutine. struct Checked { std::string key; @@ -228,14 +231,14 @@ namespace ALLuauExports std::vector names; }; constexpr size_t CHECKED_KEPT = 256; - std::mutex sCheckedMutex; + LLCoros::Mutex sCheckedMutex; std::deque sChecked; } void checked(const std::string& key, std::string_view text, std::vector names) { - const size_t hash = std::hash()(text); - const std::lock_guard lock(sCheckedMutex); + const size_t hash = std::hash()(text); + const LLCoros::LockType lock(sCheckedMutex); sChecked.erase(std::remove_if(sChecked.begin(), sChecked.end(), [&key](const Checked& one) { return one.key == key; }), sChecked.end()); sChecked.push_back({ key, text.size(), hash, std::move(names) }); if (sChecked.size() > CHECKED_KEPT) @@ -246,8 +249,8 @@ namespace ALLuauExports std::optional> checkedOf(const std::string& key, std::string_view text) { - const size_t hash = std::hash()(text); - const std::lock_guard lock(sCheckedMutex); + const size_t hash = std::hash()(text); + const LLCoros::LockType lock(sCheckedMutex); for (const Checked& one : sChecked) { if (one.key == key && one.size == text.size() && one.hash == hash) diff --git a/indra/alscript/masters/almasterindex.cpp b/indra/alscript/masters/almasterindex.cpp index 6d423ac0ec..5bda9121a8 100644 --- a/indra/alscript/masters/almasterindex.cpp +++ b/indra/alscript/masters/almasterindex.cpp @@ -31,13 +31,15 @@ #include "alserialworker.h" #include "aluploadheader.h" #include "fsyspath.h" +#include "llcoros.h" #include "llsdserialize.h" -#include #include -#include #include +#include LLCOROS_MUTEX_HEADER +#include LLCOROS_CONDVAR_HEADER + namespace { // The target a notecard's text is hashed under. @@ -61,7 +63,8 @@ namespace // all of it -- and whether one is being written now, by the writer's thread // or by the index's. One write at a time, whichever thread makes it, so // that two never meet in the file put beside the index on the way. What -// could not be written waits still, to be tried again. +// could not be written waits still, to be tried again. The index's thread +// may be the main one, and a coroutine there what waits. struct ALMasterIndex::Writer { struct Text @@ -69,13 +72,13 @@ struct ALMasterIndex::Writer std::string path; std::string text; }; - std::mutex mutex; - std::condition_variable changed; - std::optional waiting; + LLCoros::Mutex mutex; + LLCoros::ConditionVariable changed; + std::optional waiting; // Whether what waits is what a write of failed, kept for the next // drain: the next change's, which is all of it and more, or a flush's. - bool failed = false; - bool busy = false; + bool failed = false; + bool busy = false; // What waits written, and what comes meanwhile, on whichever thread // asks: one being written already by another is waited for first, so @@ -83,7 +86,7 @@ struct ALMasterIndex::Writer // that cannot be written is tried once here, said, and left waiting. void drain() { - std::unique_lock lock(mutex); + LLCoros::LockType lock(mutex); for (;;) { changed.wait(lock, [this] { return !busy; }); @@ -123,7 +126,7 @@ struct ALMasterIndex::Writer // for first, so that the answer is the write's. bool unwritten() { - std::unique_lock lock(mutex); + LLCoros::LockType lock(mutex); changed.wait(lock, [this] { return !busy; }); return failed; } @@ -534,7 +537,7 @@ void ALMasterIndex::handOver() mDirty = false; std::ostringstream out; LLSDSerialize::toPrettyXML(mLinks.toLLSD(), out); - const std::lock_guard lock(mWriter->mutex); + const LLCoros::LockType lock(mWriter->mutex); mWriter->waiting = Writer::Text{ mPath, out.str() }; mWriter->failed = false; } diff --git a/indra/alscript/preprocessor/alscriptjoblane.h b/indra/alscript/preprocessor/alscriptjoblane.h index aa426a764e..018384bde4 100644 --- a/indra/alscript/preprocessor/alscriptjoblane.h +++ b/indra/alscript/preprocessor/alscriptjoblane.h @@ -24,15 +24,17 @@ #pragma once +#include "llcoros.h" #include "llerror.h" #include #include #include #include -#include #include +#include LLCOROS_MUTEX_HEADER + // What waits for a worker, on the worker's side: it takes the next itself // as each ends, rather than waiting a frame for the main thread to hand it // over. A run before any check; a check of a script whose later check is @@ -59,7 +61,7 @@ class ALScriptJobLane // started on drain(). Nothing, once closed. bool put(Queued queued, const taking_t& taking) { - std::lock_guard guard(mLock); + LLCoros::LockType guard(mLock); if (mClosed) { return false; @@ -106,7 +108,7 @@ class ALScriptJobLane { std::function work; { - std::lock_guard guard(mLock); + LLCoros::LockType guard(mLock); mRunning.reset(); std::deque& from = !mRuns.empty() ? mRuns : mChecks; if (mClosed || from.empty()) @@ -139,7 +141,7 @@ class ALScriptJobLane // starts it again. void notStarted() { - std::lock_guard guard(mLock); + LLCoros::LockType guard(mLock); mDraining = false; mRuns.clear(); mChecks.clear(); @@ -148,7 +150,7 @@ class ALScriptJobLane // Nothing more taken, and what waits let go of; kept closed. void close() { - std::lock_guard guard(mLock); + LLCoros::LockType guard(mLock); mClosed = true; mRuns.clear(); mChecks.clear(); @@ -156,12 +158,13 @@ class ALScriptJobLane size_t waiting() { - std::lock_guard guard(mLock); + LLCoros::LockType guard(mLock); return mRuns.size() + mChecks.size(); } private: - std::mutex mLock; + // The main thread's side may be a coroutine's. + LLCoros::Mutex mLock; std::deque mRuns; std::deque mChecks; bool mDraining = false; diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 1ec792fb39..897a0c1664 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -134,6 +134,7 @@ target_sources( alsanitizer.h alsavehistory.h alscratcharena.h + alsharedmutex.h alsimd.h alwatchedfile.h always_return.h @@ -316,6 +317,7 @@ if(AL_BUILD_TESTS) tests/alsavehistory_test.cpp tests/alscratcharena_test.cpp tests/alserialworker_test.cpp + tests/alsharedmutex_test.cpp tests/alsimd_test.cpp tests/alwatchedfile_test.cpp tests/apply_test.cpp diff --git a/indra/llcommon/alparamtype.cpp b/indra/llcommon/alparamtype.cpp index a075ed6b54..f6ee594e00 100644 --- a/indra/llcommon/alparamtype.cpp +++ b/indra/llcommon/alparamtype.cpp @@ -26,9 +26,11 @@ #include "alparamtype.h" +#include "llcoros.h" + #include -#include +#include LLCOROS_MUTEX_HEADER namespace { @@ -42,9 +44,11 @@ namespace return sTable; } - std::mutex& lock() + // A block is first made wherever it is first wanted, a coroutine + // among those places. + LLCoros::Mutex& lock() { - static std::mutex sMutex; + static LLCoros::Mutex sMutex; return sMutex; } } @@ -52,14 +56,14 @@ namespace //static void ALParamTypes::record(const LLInitParam::ParamDescriptor* param, const ALParamType& type) { - std::lock_guard guard(lock()); + LLCoros::LockType guard(lock()); table()[param] = type; } //static const ALParamType* ALParamTypes::find(const LLInitParam::ParamDescriptor* param) { - std::lock_guard guard(lock()); + LLCoros::LockType guard(lock()); const table_t::const_iterator found = table().find(param); return found == table().end() ? nullptr : &found->second; } @@ -67,14 +71,14 @@ const ALParamType* ALParamTypes::find(const LLInitParam::ParamDescriptor* param) //static size_t ALParamTypes::count() { - std::lock_guard guard(lock()); + LLCoros::LockType guard(lock()); return table().size(); } //static size_t ALParamTypes::bytes() { - std::lock_guard guard(lock()); + LLCoros::LockType guard(lock()); // A flat map's slot is the pair plus a byte of control, and it keeps room // it has not filled: the load factor is what the capacity says. return (sizeof(table_t::value_type) + 1) * table().bucket_count(); diff --git a/indra/llcommon/alrecoverystore.cpp b/indra/llcommon/alrecoverystore.cpp index e6f4b9d77c..8fc5e17ce0 100644 --- a/indra/llcommon/alrecoverystore.cpp +++ b/indra/llcommon/alrecoverystore.cpp @@ -28,6 +28,7 @@ #include "alfilewrite.h" #include "fsyspath.h" +#include "llcoros.h" #include "llfile.h" #include "llsdserialize.h" #include "lltimer.h" @@ -35,15 +36,16 @@ #include #include -#include #include #include #include #include -#include #include #include +#include LLCOROS_MUTEX_HEADER +#include LLCOROS_CONDVAR_HEADER + namespace { const char* const EXTENSION = ".llsd"; @@ -294,7 +296,8 @@ F64 ALRecoveryRetry::delayAfter(S32 failures) // with where it goes -- handed over whole, what it holds of LLSD made on // the thread that asked and held by nothing else, since an LLSD's count of // who holds it is no thread's but one's -- whether one is being written -// now, and whether the writer has been asked to write them. +// now, and whether the writer has been asked to write them. Handed over and +// waited for on the main thread, which a coroutine may do. struct ALRecoveryStore::Writer { struct Waiting @@ -303,8 +306,8 @@ struct ALRecoveryStore::Writer ALRecoveryEntry entry; bool durable = false; }; - std::mutex mutex; - std::condition_variable changed; + LLCoros::Mutex mutex; + LLCoros::ConditionVariable changed; std::map waiting; bool busy = false; bool posted = false; @@ -313,7 +316,7 @@ struct ALRecoveryStore::Writer // Everything waiting written, what comes meanwhile too. void drain() { - std::unique_lock lock(mutex); + LLCoros::LockType lock(mutex); while (!waiting.empty()) { auto one = waiting.extract(waiting.begin()); @@ -396,7 +399,7 @@ void ALRecoveryStore::writeSoon(ALRecoveryEntry entry, bool durable) mWriter = std::make_unique(); } { - const std::lock_guard lock(mWriter->mutex); + const LLCoros::LockType lock(mWriter->mutex); // A durable write still waiting stays durable when a newer one // takes its place. if (const auto was = mWriter->waiting.find(key); was != mWriter->waiting.end()) @@ -425,7 +428,7 @@ void ALRecoveryStore::flush() const { return; } - std::unique_lock lock(mWriter->mutex); + LLCoros::LockType lock(mWriter->mutex); mWriter->changed.wait(lock, [this] { return mWriter->waiting.empty() && !mWriter->busy && !mWriter->posted; }); } @@ -434,7 +437,7 @@ std::vector ALRecoveryStore::takeFailures() std::vector out; if (mWriter) { - const std::lock_guard lock(mWriter->mutex); + const LLCoros::LockType lock(mWriter->mutex); out.swap(mWriter->failed); } return out; @@ -446,10 +449,10 @@ bool ALRecoveryStore::takeFailure(const std::string& key) { return false; } - const std::lock_guard lock(mWriter->mutex); - std::vector& failed = mWriter->failed; - const auto gone = std::remove(failed.begin(), failed.end(), key); - const bool any = gone != failed.end(); + const LLCoros::LockType lock(mWriter->mutex); + std::vector& failed = mWriter->failed; + const auto gone = std::remove(failed.begin(), failed.end(), key); + const bool any = gone != failed.end(); failed.erase(gone, failed.end()); return any; } diff --git a/indra/llcommon/alsavehistory.cpp b/indra/llcommon/alsavehistory.cpp index 943c3042a1..67195332a6 100644 --- a/indra/llcommon/alsavehistory.cpp +++ b/indra/llcommon/alsavehistory.cpp @@ -219,7 +219,7 @@ std::vector ALSaveHistory::listLocked(const std::string& key) const bool ALSaveHistory::keep(ALSavedText saved) { LL_PROFILE_ZONE_SCOPED_CATEGORY_SCRIPTDEV; - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); if (saved.key.empty()) { return false; @@ -255,19 +255,19 @@ bool ALSaveHistory::keep(ALSavedText saved) std::vector ALSaveHistory::list(const std::string& key) const { - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); return listLocked(key); } size_t ALSaveHistory::count(const std::string& key) const { - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); return key.empty() ? 0 : savesIn(folderOf(key)).size(); } bool ALSaveHistory::load(ALSavedText& saved) const { - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); ALSavedText read; if (saved.path.empty() || !readSaved(saved.path, read, true)) { @@ -280,7 +280,7 @@ bool ALSaveHistory::load(ALSavedText& saved) const bool ALSaveHistory::holds(const std::string& key, const LLUUID& asset, const std::string& text, size_t newest) const { - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); if (key.empty() || newest == 0) { return false; @@ -306,7 +306,7 @@ bool ALSaveHistory::holds(const std::string& key, const LLUUID& asset, const std bool ALSaveHistory::rekey(const std::string& from, const std::string& to) { - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); if (from == to || from.empty() || to.empty()) { return false; @@ -360,7 +360,7 @@ void ALSaveHistory::trim(const std::string& folder, const LLDate& now) void ALSaveHistory::prune(const LLDate& now) { LL_PROFILE_ZONE_SCOPED_CATEGORY_SCRIPTDEV; - std::lock_guard lock(mLock); + LLCoros::LockType lock(mLock); struct Save { S64 at = 0; diff --git a/indra/llcommon/alsavehistory.h b/indra/llcommon/alsavehistory.h index 3c914e6dbc..62fbdc4ead 100644 --- a/indra/llcommon/alsavehistory.h +++ b/indra/llcommon/alsavehistory.h @@ -24,14 +24,16 @@ #pragma once +#include "llcoros.h" #include "lldate.h" #include "llsd.h" #include "lluuid.h" -#include #include #include +#include LLCOROS_MUTEX_HEADER + // A save of a script or a notecard as it went up: its text, and enough // about whose it was to list it and to say what it was. struct ALSavedText @@ -118,9 +120,10 @@ class ALSaveHistory // A key's saves past the count, or older than the age, let go of. void trim(const std::string& folder, const LLDate& now); - std::string mDirectory; - size_t mPerKey = MAX_PER_KEY; - F64 mMaxAge = MAX_AGE; - size_t mBytes = MAX_BYTES; - mutable std::mutex mLock; + std::string mDirectory; + size_t mPerKey = MAX_PER_KEY; + F64 mMaxAge = MAX_AGE; + size_t mBytes = MAX_BYTES; + // Listed on the main thread, which a coroutine may do. + mutable LLCoros::Mutex mLock; }; diff --git a/indra/llcommon/alsharedmutex.h b/indra/llcommon/alsharedmutex.h new file mode 100644 index 0000000000..3bc7ab071e --- /dev/null +++ b/indra/llcommon/alsharedmutex.h @@ -0,0 +1,123 @@ +/** + * @file alsharedmutex.h + * @brief A reader-writer lock that lets a waiting writer in ahead of the + * readers that come after it. + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Alchemy Viewer Source Code + * Copyright (C) 2026, Rye + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * $/LicenseInfo$ + */ + +#pragma once + +#include "stdtypes.h" + +#include +#include + +#if defined(__GLIBC__) +#include +#endif + +// A reader-writer lock neither side may take again while it holds it. A writer +// that has to wait is let in ahead of the readers that come after it, so readers +// that keep overlapping cannot keep it out. It takes std::unique_lock and +// std::shared_lock. LLSharedMutex, which either side may re-enter, is built on it. +class ALSharedMutex +{ +public: +#if defined(__GLIBC__) + // glibc's default rwlock lets a new reader in past a waiting writer, so readers that + // keep overlapping keep a writer out for as long as they do. This one prefers writers, + // as libc++'s shared_mutex does. glibc calls that kind nonrecursive: a reader taking + // it again behind a waiting writer would wait on itself. + ALSharedMutex() + { + pthread_rwlockattr_t attr; + pthread_rwlockattr_init(&attr); + pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + pthread_rwlock_init(&mLock, &attr); + pthread_rwlockattr_destroy(&attr); + } + ~ALSharedMutex() { pthread_rwlock_destroy(&mLock); } +#else + ALSharedMutex() = default; +#endif + + ALSharedMutex(const ALSharedMutex&) = delete; + ALSharedMutex& operator=(const ALSharedMutex&) = delete; + +#if defined(__GLIBC__) + void lock() { pthread_rwlock_wrlock(&mLock); } + void unlock() { pthread_rwlock_unlock(&mLock); } + void lock_shared() { pthread_rwlock_rdlock(&mLock); } + void unlock_shared() { pthread_rwlock_unlock(&mLock); } +#elif defined(LL_WINDOWS) + // std::shared_mutex is an SRW lock, which lets in any thread that finds it free: a + // writer it wakes can find it taken again before it runs, and readers that keep + // overlapping kept one out for over 10 s. A writer that has to wait shuts a gate + // first, and new readers wait at the gate instead of at the lock, so only a reader + // already past it can come between the writer and the lock, and only once. + void lock() + { + if (mLock.try_lock()) + { + return; + } + // The count only sends readers to the gate and never decides who holds the + // lock, so relaxed suffices: a reader that misses a rise gets in once, as it + // would have a moment earlier. + mWritersWaiting.fetch_add(1, std::memory_order_relaxed); + mGate.lock(); + mLock.lock(); + mGate.unlock(); + mWritersWaiting.fetch_sub(1, std::memory_order_relaxed); + } + void unlock() { mLock.unlock(); } + + void lock_shared() + { + if (mWritersWaiting.load(std::memory_order_relaxed) != 0) + { + // Shut until the waiting writer has the lock. + mGate.lock_shared(); + mGate.unlock_shared(); + } + mLock.lock_shared(); + } + void unlock_shared() { mLock.unlock_shared(); } +#else + // macOS's: libc++'s shared_mutex already lets a waiting writer in first. + void lock() { mLock.lock(); } + void unlock() { mLock.unlock(); } + void lock_shared() { mLock.lock_shared(); } + void unlock_shared() { mLock.unlock_shared(); } +#endif + +private: +#if defined(__GLIBC__) + pthread_rwlock_t mLock; +#elif defined(LL_WINDOWS) + std::shared_mutex mLock; + // An SRW lock too: readers pass it together, and std::mutex is ten times its size. + std::shared_mutex mGate; + std::atomic mWritersWaiting{0}; +#else + std::shared_mutex mLock; +#endif +}; diff --git a/indra/llcommon/llinitparam.cpp b/indra/llcommon/llinitparam.cpp index 2e68dd52ed..2aeed0eb99 100644 --- a/indra/llcommon/llinitparam.cpp +++ b/indra/llcommon/llinitparam.cpp @@ -28,15 +28,17 @@ #include "linden_common.h" #include "llinitparam.h" +#include "llcoros.h" #include "llformat.h" #include #include #include -#include #include #include +#include LLCOROS_MUTEX_HEADER + namespace LLInitParam { @@ -135,16 +137,16 @@ namespace LLInitParam // // The lock is not for contention -- a descriptor is allocated once, // during a block type's first construction -- but because which thread - // that happens on is not ours to say. + // that happens on is not ours to say, nor whether it is in a coroutine. constexpr std::size_t DESCRIPTORS_PER_CHUNK = 64; ParamDescriptor* allocateDescriptor() { - static std::mutex sChunkMutex; + static LLCoros::Mutex sChunkMutex; static std::vector > sChunks; static std::size_t sUsedInChunk = DESCRIPTORS_PER_CHUNK; - std::lock_guard lock(sChunkMutex); + LLCoros::LockType lock(sChunkMutex); if (sUsedInChunk == DESCRIPTORS_PER_CHUNK) { sChunks.push_back(std::make_unique(DESCRIPTORS_PER_CHUNK)); diff --git a/indra/llcommon/llmutex.h b/indra/llcommon/llmutex.h index 249f269b57..4988b11dac 100644 --- a/indra/llcommon/llmutex.h +++ b/indra/llcommon/llmutex.h @@ -28,6 +28,7 @@ #define LL_LLMUTEX_H #include "stdtypes.h" +#include "alsharedmutex.h" #include "llthread.h" #include @@ -37,10 +38,6 @@ #include #include -#if defined(__GLIBC__) -#include -#endif - //============================================================================ class LL_COMMON_API LLMutex @@ -95,84 +92,7 @@ class LL_COMMON_API LLSharedMutex template void unlock(); private: -#if defined(__GLIBC__) - // glibc's default rwlock lets a new reader in past a waiting writer, so readers that - // keep overlapping keep a writer out for as long as they do. This one prefers writers, - // as libc++'s shared_mutex does. glibc calls that kind nonrecursive: a reader taking - // it again behind a waiting writer would wait on itself, which a shared re-entry here - // never does. - class RWLock - { - public: - RWLock() - { - pthread_rwlockattr_t attr; - pthread_rwlockattr_init(&attr); - pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - pthread_rwlock_init(&mLock, &attr); - pthread_rwlockattr_destroy(&attr); - } - ~RWLock() { pthread_rwlock_destroy(&mLock); } - - RWLock(const RWLock&) = delete; - RWLock& operator=(const RWLock&) = delete; - - void lock() { pthread_rwlock_wrlock(&mLock); } - void unlock() { pthread_rwlock_unlock(&mLock); } - void lock_shared() { pthread_rwlock_rdlock(&mLock); } - void unlock_shared() { pthread_rwlock_unlock(&mLock); } - - private: - pthread_rwlock_t mLock; - }; -#elif defined(LL_WINDOWS) - // std::shared_mutex is an SRW lock, which lets in any thread that finds it free: a - // writer it wakes can find it taken again before it runs, and readers that keep - // overlapping kept one out for over 10 s. A writer that has to wait shuts a gate - // first, and new readers wait at the gate instead of at the lock, so only a reader - // already past it can come between the writer and the lock, and only once. - class RWLock - { - public: - void lock() - { - if (mLock.try_lock()) - { - return; - } - // The count only sends readers to the gate and never decides who holds the - // lock, so relaxed suffices: a reader that misses a rise gets in once, as it - // would have a moment earlier. - mWritersWaiting.fetch_add(1, std::memory_order_relaxed); - mGate.lock(); - mLock.lock(); - mGate.unlock(); - mWritersWaiting.fetch_sub(1, std::memory_order_relaxed); - } - void unlock() { mLock.unlock(); } - - void lock_shared() - { - if (mWritersWaiting.load(std::memory_order_relaxed) != 0) - { - // Shut until the waiting writer has the lock. - mGate.lock_shared(); - mGate.unlock_shared(); - } - mLock.lock_shared(); - } - void unlock_shared() { mLock.unlock_shared(); } - - private: - std::shared_mutex mLock; - // An SRW lock too: readers pass it together, and std::mutex is ten times its size. - std::shared_mutex mGate; - std::atomic mWritersWaiting{0}; - }; -#else - using RWLock = std::shared_mutex; -#endif - RWLock mSharedMutex; + ALSharedMutex mSharedMutex; // The exclusive holder. Relaxed for the reason LLMutex::mLockingThread is. std::atomic mExclusiveOwner; // Owner-only: the holds the exclusive holder has taken beyond its first, either way. diff --git a/indra/llcommon/llprocess.cpp b/indra/llcommon/llprocess.cpp index fc0e034db4..f50fb30ba8 100644 --- a/indra/llcommon/llprocess.cpp +++ b/indra/llcommon/llprocess.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -65,30 +66,28 @@ #if LL_WINDOWS #include "llwin32headers.h" -#include +#include "llcoros.h" +#include LLCOROS_MUTEX_HEADER namespace { // Global job object that will kill all child processes when parent terminates HANDLE g_jobObject = NULL; - bool g_jobObjectInitialized = false; - std::mutex g_jobObjectMutex; - - void InitializeJobObject() + // Set once g_jobObject holds the job, or NULL for good, so that a launch + // that finds it set may read g_jobObject without the lock. + std::atomic g_jobObjectInitialized{false}; + // A child may be launched from a coroutine. + LLCoros::Mutex g_jobObjectMutex; + + // A job that kills every process in it when its last handle closes, or + // NULL where one could not be made. + HANDLE CreateKillOnCloseJob() { - if (g_jobObjectInitialized) - return; - std::lock_guard lock(g_jobObjectMutex); - if (g_jobObjectInitialized) - return; - - g_jobObjectInitialized = true; - // Create a job object - g_jobObject = ::CreateJobObjectW(NULL, NULL); - if (g_jobObject == NULL) + HANDLE job = ::CreateJobObjectW(NULL, NULL); + if (job == NULL) { LL_WARNS("LLProcess") << "Failed to create job object: " << ::GetLastError() << LL_ENDL; - return; + return NULL; } // Configure the job to kill all processes when the last handle closes @@ -99,22 +98,34 @@ namespace { // Windows 8+ supports nested jobs (for VS studio) jeli.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK; - if (!::SetInformationJobObject(g_jobObject, JobObjectExtendedLimitInformation, + if (!::SetInformationJobObject(job, JobObjectExtendedLimitInformation, &jeli, sizeof(jeli))) { LL_WARNS("LLProcess") << "Failed to set job object limits: " << ::GetLastError() << LL_ENDL; - ::CloseHandle(g_jobObject); - g_jobObject = NULL; - return; + ::CloseHandle(job); + return NULL; } LL_INFOS("LLProcess") << "Job object created - child processes will terminate with parent" << LL_ENDL; + return job; + } + + void InitializeJobObject() + { + if (g_jobObjectInitialized.load(std::memory_order_acquire)) + return; + LLCoros::LockType lock(g_jobObjectMutex); + if (g_jobObjectInitialized.load(std::memory_order_relaxed)) + return; + + // Tried once, whether or not it is made. + g_jobObject = CreateKillOnCloseJob(); + g_jobObjectInitialized.store(true, std::memory_order_release); } void AssignProcessToJob(HANDLE hProcess, const std::string& desc) { - if (!g_jobObjectInitialized) - InitializeJobObject(); + InitializeJobObject(); if (g_jobObject != NULL) { diff --git a/indra/llcommon/llstringtable.cpp b/indra/llcommon/llstringtable.cpp index a1a5223744..9088b03541 100644 --- a/indra/llcommon/llstringtable.cpp +++ b/indra/llcommon/llstringtable.cpp @@ -55,13 +55,13 @@ LLStringTable::~LLStringTable() = default; S32 LLStringTable::getUniqueEntries() const { - std::shared_lock lock(mMutex); + std::shared_lock lock(mMutex); return static_cast(mTable.size()); } LLStringTableEntry* LLStringTable::checkStringEntry(std::string_view str) const { - std::shared_lock lock(mMutex); + std::shared_lock lock(mMutex); table_t::const_iterator found = mTable.find(str); return found == mTable.end() ? nullptr : found->second.get(); } @@ -88,7 +88,7 @@ LLStringTableEntry* LLStringTable::addStringEntry(std::string_view str) // of tag and attribute names recur across every file parsed -- so look // first under a shared lock and only take the table exclusively to insert. { - std::shared_lock lock(mMutex); + std::shared_lock lock(mMutex); table_t::const_iterator found = mTable.find(str); if (found != mTable.end()) { @@ -100,7 +100,7 @@ LLStringTableEntry* LLStringTable::addStringEntry(std::string_view str) } } - std::unique_lock lock(mMutex); + std::unique_lock lock(mMutex); // Another thread may have inserted it while the lock was not held. table_t::const_iterator found = mTable.find(str); @@ -147,7 +147,7 @@ void LLStringTable::removeString(std::string_view str) return; } - std::unique_lock lock(mMutex); + std::unique_lock lock(mMutex); table_t::iterator found = mTable.find(str); if (found != mTable.end() && !found->second->decCount()) { diff --git a/indra/llcommon/llstringtable.h b/indra/llcommon/llstringtable.h index 4b0bf04c51..df40e8a1f0 100644 --- a/indra/llcommon/llstringtable.h +++ b/indra/llcommon/llstringtable.h @@ -28,6 +28,7 @@ #ifndef LL_STRING_TABLE_H #define LL_STRING_TABLE_H +#include "alsharedmutex.h" #include "lldefs.h" #include "llformat.h" #include "llstl.h" @@ -35,7 +36,6 @@ #include #include #include -#include #include #include @@ -127,7 +127,7 @@ class LL_COMMON_API LLStringTable ll::string_hash, std::equal_to<>> table_t; - mutable std::shared_mutex mMutex; + mutable ALSharedMutex mMutex; table_t mTable; const bool mCounted; }; diff --git a/indra/llcommon/llwatchdog.cpp b/indra/llcommon/llwatchdog.cpp index 53fe88b7a5..e8e5c09f6b 100644 --- a/indra/llcommon/llwatchdog.cpp +++ b/indra/llcommon/llwatchdog.cpp @@ -125,38 +125,38 @@ LLWatchdogTimeout::~LLWatchdogTimeout() bool LLWatchdogTimeout::isAlive() const { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); return (mTimer.getStarted() && !mTimer.hasExpired()); } bool LLWatchdogTimeout::hasExpired() const { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); return mTimer.hasExpired(); } bool LLWatchdogTimeout::started() const { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); return mTimer.getStarted(); } void LLWatchdogTimeout::reset() { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); mTimer.setTimerExpirySec(mTimeout); } void LLWatchdogTimeout::setTimeout(F32 d) { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); mTimeout = d; } void LLWatchdogTimeout::start(std::string_view state) { { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); if (mTimeout == 0) { LL_WARNS() << "Cant' start watchdog entry - no timeout set" << LL_ENDL; @@ -177,13 +177,13 @@ void LLWatchdogTimeout::start(std::string_view state) void LLWatchdogTimeout::stop() { LLWatchdogEntry::stop(); - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); mTimer.stop(); } void LLWatchdogTimeout::ping(std::string_view state) { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); if (!state.empty()) { mPingState = state; @@ -193,7 +193,7 @@ void LLWatchdogTimeout::ping(std::string_view state) std::string LLWatchdogTimeout::getLastState() const { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mMutex); return mPingState; } diff --git a/indra/llcommon/llwatchdog.h b/indra/llcommon/llwatchdog.h index 168cb9e48a..3849006f9b 100644 --- a/indra/llcommon/llwatchdog.h +++ b/indra/llcommon/llwatchdog.h @@ -34,7 +34,6 @@ #include "llsingleton.h" #include -#include // LLWatchdogEntry is the interface used by the tasks that // need to be watched. @@ -86,7 +85,7 @@ class LL_COMMON_API LLWatchdogTimeout : public LLWatchdogEntry // The watched thread pings and the watchdog thread checks, and resets // when it was itself held up: the timer, the timeout and the state are // read and written under this. - mutable std::mutex mMutex; + mutable LLMutex mMutex; LLTimer mTimer; F32 mTimeout; std::string mPingState; diff --git a/indra/llcommon/tests/alserialworker_test.cpp b/indra/llcommon/tests/alserialworker_test.cpp index ebf8df2572..59597923ea 100644 --- a/indra/llcommon/tests/alserialworker_test.cpp +++ b/indra/llcommon/tests/alserialworker_test.cpp @@ -28,13 +28,13 @@ #include "../alserialworker.h" #include "../llevents.h" +#include "../llmutex.h" #include "../llsdutil.h" #include #include #include -#include #include #include @@ -43,13 +43,13 @@ namespace struct alserialworker_data { // What the jobs did, in the order they did it, and where. - std::mutex mutex; + LLMutex mutex; std::vector done; std::vector where; void note(int n) { - const std::lock_guard lock(mutex); + LLMutexLock lock(&mutex); done.push_back(n); where.push_back(std::this_thread::get_id()); } @@ -60,7 +60,7 @@ namespace for (int tries = 0; tries < 2000; ++tries) { { - const std::lock_guard lock(mutex); + LLMutexLock lock(&mutex); if (done.size() >= count) { return true; diff --git a/indra/llcommon/tests/alsharedmutex_test.cpp b/indra/llcommon/tests/alsharedmutex_test.cpp new file mode 100644 index 0000000000..68b61f3783 --- /dev/null +++ b/indra/llcommon/tests/alsharedmutex_test.cpp @@ -0,0 +1,201 @@ +/** + * @file alsharedmutex_test.cpp + * @brief Unit tests for ALSharedMutex's sharing and its preference for writers + * + * Copyright (c) 2026, Rye + * + * The source code in this file is provided to you under the terms of the + * GNU Lesser General Public License, version 2.1, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. Terms of the LGPL can be found in doc/LGPL-licence.txt + * in this distribution, or online at http://www.gnu.org/licenses/lgpl-2.1.txt + * + */ + +#include "linden_common.h" + +#include "alsharedmutex.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace +{ + using namespace std::chrono_literals; + + // Runs a body on a thread of its own and says whether it has returned. A body + // still blocked when the worker goes out of scope is left to finish without it: + // a deadlock the test has already reported, or a probe kept out of a lock the + // test still holds. The thread owns the flag it sets, and a probe owns what it + // locks, so both outlive the worker. + class Worker + { + public: + template + explicit Worker(FUNC&& body) + : mDone(std::make_shared>(false)) + , mThread([done = mDone, body = std::forward(body)]() mutable + { + body(); + *done = true; + }) + { + } + + ~Worker() + { + if (*mDone) + { + mThread.join(); + } + else + { + mThread.detach(); + } + } + + bool finishesWithin(std::chrono::milliseconds limit) + { + const auto deadline = std::chrono::steady_clock::now() + limit; + while (!*mDone) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(1ms); + } + return true; + } + + private: + std::shared_ptr> mDone; + std::thread mThread; + }; + + // Long enough that a thread which could run has run; a body that has not + // returned by then is blocked. + constexpr auto BLOCKED = 50ms; + // Long enough that a thread which was released has finished. + constexpr auto RELEASED = 5s; + + // The lock and what the probes record, held by every probe as well as the test, + // since a blocked probe outlives it. + struct Shared + { + ALSharedMutex mMutex; + std::atomic mTurns{0}; + std::atomic mWriterTurn{0}; + std::atomic mReaderTurn{0}; + }; + + struct alsharedmutex_data + { + std::shared_ptr mShared = std::make_shared(); + ALSharedMutex& mMutex = mShared->mMutex; + + Worker writer() + { + return Worker([shared = mShared]() + { + std::unique_lock lock(shared->mMutex); + shared->mWriterTurn = ++shared->mTurns; + }); + } + + Worker reader() + { + return Worker([shared = mShared]() + { + std::shared_lock lock(shared->mMutex); + shared->mReaderTurn = ++shared->mTurns; + }); + } + }; + + TEST_CASE_METHOD(alsharedmutex_data, "alsharedmutex: readers share and a writer waits for them", "[alsharedmutex]") + { + std::shared_lock held(mMutex); + + Worker second = reader(); + UNSCOPED_INFO("a second reader waited on the first"); + CHECK(second.finishesWithin(RELEASED)); + + Worker waiting = writer(); + CHECK_FALSE(waiting.finishesWithin(BLOCKED)); + + held.unlock(); + CHECK(waiting.finishesWithin(RELEASED)); + } + + TEST_CASE_METHOD(alsharedmutex_data, "alsharedmutex: a writer excludes readers and writers", "[alsharedmutex]") + { + std::unique_lock held(mMutex); + + Worker waiting_reader = reader(); + Worker waiting_writer = writer(); + CHECK_FALSE(waiting_reader.finishesWithin(BLOCKED)); + CHECK_FALSE(waiting_writer.finishesWithin(BLOCKED)); + + held.unlock(); + CHECK(waiting_reader.finishesWithin(RELEASED)); + CHECK(waiting_writer.finishesWithin(RELEASED)); + } + + TEST_CASE_METHOD(alsharedmutex_data, "alsharedmutex: a reader that comes while a writer waits goes after it", "[alsharedmutex]") + { + // glibc's default rwlock, which is std::shared_mutex there, lets the late reader + // straight in beside the first. + std::shared_lock held(mMutex); + + Worker waiting = writer(); + CHECK_FALSE(waiting.finishesWithin(BLOCKED)); + + Worker late = reader(); + UNSCOPED_INFO("the late reader got in past the waiting writer"); + CHECK_FALSE(late.finishesWithin(BLOCKED)); + + held.unlock(); + CHECK(waiting.finishesWithin(RELEASED)); + CHECK(late.finishesWithin(RELEASED)); + UNSCOPED_INFO("the late reader went before the writer"); + CHECK(mShared->mWriterTurn < mShared->mReaderTurn); + } + + TEST_CASE_METHOD(alsharedmutex_data, "alsharedmutex: a writer gets in while readers keep overlapping", "[alsharedmutex]") + { + // A lock that lets new readers in past a waiting writer keeps the writer out for + // as long as the readers overlap. The SRW lock lets them in only when it wakes + // the writer, so under load it fails here now and then. + auto stop = std::make_shared>(false); + auto read = [shared = mShared, stop]() + { + while (!*stop) + { + std::shared_lock lock(shared->mMutex); + std::this_thread::sleep_for(1ms); + } + }; + Worker reader1(read); + Worker reader2(read); + Worker reader3(read); + Worker reader4(read); + std::this_thread::sleep_for(BLOCKED); + + Worker waiting = writer(); + UNSCOPED_INFO("the readers kept the writer out"); + CHECK(waiting.finishesWithin(RELEASED)); + + *stop = true; + CHECK(reader1.finishesWithin(RELEASED)); + CHECK(reader2.finishesWithin(RELEASED)); + CHECK(reader3.finishesWithin(RELEASED)); + CHECK(reader4.finishesWithin(RELEASED)); + } +} diff --git a/indra/llcommon/tests/llmainthreadtask_test.cpp b/indra/llcommon/tests/llmainthreadtask_test.cpp index ff4fde68dd..010b6baed7 100644 --- a/indra/llcommon/tests/llmainthreadtask_test.cpp +++ b/indra/llcommon/tests/llmainthreadtask_test.cpp @@ -22,6 +22,7 @@ #include "../test/sync.h" #include "llthread.h" // on_main_thread() #include "lleventtimer.h" +#include "llmutex.h" #include "lockstatic.h" /***************************************************************************** @@ -58,7 +59,7 @@ namespace // in an anonymous namespace, so the name is this test's own. struct MainThreadTaskStatic { - std::mutex mMutex; // LockStatic looks for mMutex + LLMutex mMutex; // LockStatic looks for mMutex bool ran{false}; }; typedef llthread::LockStatic LockStatic; diff --git a/indra/llcorehttp/_mutex.h b/indra/llcorehttp/_mutex.h index 43782d3a37..d17580dc44 100644 --- a/indra/llcorehttp/_mutex.h +++ b/indra/llcorehttp/_mutex.h @@ -27,26 +27,31 @@ #ifndef LLCOREINT_MUTEX_H_ #define LLCOREINT_MUTEX_H_ -#include -#include +#include "llcoros.h" +#include LLCOROS_MUTEX_HEADER +#include LLCOROS_CONDVAR_HEADER namespace LLCoreInt { +// Requests are queued from coroutines on the main thread, and the worker +// waits on the queue: a coroutine that has to wait for one of these parks +// alone rather than holding up its thread. + // MUTEX TYPES // unique mutex type -typedef std::mutex HttpMutex; +typedef LLCoros::Mutex HttpMutex; // CONDITION VARIABLES // standard condition variable -typedef std::condition_variable HttpConditionVariable; +typedef LLCoros::ConditionVariable HttpConditionVariable; // LOCKS AND FENCES // scoped unique lock -typedef std::unique_lock HttpScopedLock; +typedef LLCoros::LockType HttpScopedLock; } diff --git a/indra/llcorehttp/httpstats.cpp b/indra/llcorehttp/httpstats.cpp index f64752e91e..eb438c1863 100644 --- a/indra/llcorehttp/httpstats.cpp +++ b/indra/llcorehttp/httpstats.cpp @@ -41,7 +41,7 @@ HTTPStats::~HTTPStats() void HTTPStats::resetStats() { - std::lock_guard lock(mStatsMutex); + LLCoros::LockType lock(mStatsMutex); mResutCodes.clear(); mDataDown.reset(); mDataUp.reset(); @@ -51,7 +51,7 @@ void HTTPStats::resetStats() void HTTPStats::recordResultCode(S32 code) { - std::lock_guard lock(mStatsMutex); + LLCoros::LockType lock(mStatsMutex); std::map::iterator it; it = mResutCodes.find(code); @@ -88,7 +88,7 @@ namespace void HTTPStats::dumpStats() { - std::lock_guard lock(mStatsMutex); + LLCoros::LockType lock(mStatsMutex); std::stringstream out; out << "HTTP DATA SUMMARY" << std::endl; diff --git a/indra/llcorehttp/httpstats.h b/indra/llcorehttp/httpstats.h index 08a537cda7..3da41eed32 100644 --- a/indra/llcorehttp/httpstats.h +++ b/indra/llcorehttp/httpstats.h @@ -29,10 +29,13 @@ #include "lltracerecording.h" #include "lltrace.h" +#include "llcoros.h" #include "llstatsaccumulator.h" #include "llsingleton.h" #include "llsd.h" +#include LLCOROS_MUTEX_HEADER + namespace LLCore { class HTTPStats final : public LLSimpleton @@ -47,19 +50,19 @@ namespace LLCore void recordDataDown(size_t bytes) { - std::lock_guard lock(mStatsMutex); + LLCoros::LockType lock(mStatsMutex); mDataDown.push((F32)bytes); } void recordDataUp(size_t bytes) { - std::lock_guard lock(mStatsMutex); + LLCoros::LockType lock(mStatsMutex); mDataUp.push((F32)bytes); } void recordHTTPRequest() { - std::lock_guard lock(mStatsMutex); + LLCoros::LockType lock(mStatsMutex); ++mRequests; } @@ -74,7 +77,8 @@ namespace LLCore std::map mResutCodes; - std::mutex mStatsMutex; + // Taken where a request is made, which a coroutine may do. + LLCoros::Mutex mStatsMutex; }; diff --git a/indra/llrender/llvertexbuffer.cpp b/indra/llrender/llvertexbuffer.cpp index ad3f63b036..e0b10aae55 100644 --- a/indra/llrender/llvertexbuffer.cpp +++ b/indra/llrender/llvertexbuffer.cpp @@ -77,6 +77,8 @@ struct CompareMappedRegion #if ENABLE_GL_WORK_QUEUE +#include "llmutex.h" + #define THREAD_COUNT 1 //============================================================================ @@ -111,8 +113,8 @@ class GLWorkQueue void syncGL(); private: - std::mutex mMutex; - std::condition_variable mCondition; + // Guards mQueue and mClosed, and the workers sleep on it. + LLCondition mCondition; std::queue mQueue; bool mClosed = false; }; @@ -126,7 +128,7 @@ void GLWorkQueue::syncGL() { /*if (mSync) { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mCondition); glWaitSync(mSync, 0, GL_TIMEOUT_IGNORED); mSync = 0; }*/ @@ -135,7 +137,7 @@ void GLWorkQueue::syncGL() size_t GLWorkQueue::size() { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; - std::lock_guard lock(mMutex); + LLMutexLock lock(&mCondition); return mQueue.size(); } @@ -148,42 +150,35 @@ void GLWorkQueue::post(const GLWorkQueue::Work& value) { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mCondition); mQueue.push(std::move(value)); } - mCondition.notify_one(); + mCondition.signal(); } // Get the next element from the queue GLWorkQueue::Work GLWorkQueue::pop() { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; - // Lock the mutex - { - std::unique_lock lock(mMutex); + Work ret; - // Wait for a new element to become available or for the queue to close + // Wait for a new element to become available or for the queue to close, + // and take the next element under the same hold + mCondition.wait([&] { - mCondition.wait(lock, [=] { return !mQueue.empty() || mClosed; }); - } - } - - Work ret; + if (mQueue.size() > 0) + { + ret = mQueue.front(); + mQueue.pop(); + return true; + } + return mClosed; + }); + if (!ret) { - std::lock_guard lock(mMutex); - - // Get the next element from the queue - if (mQueue.size() > 0) - { - ret = mQueue.front(); - mQueue.pop(); - } - else - { - ret = []() {}; - } + ret = []() {}; } return ret; @@ -209,17 +204,17 @@ void GLWorkQueue::close() { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; { - std::lock_guard lock(mMutex); + LLMutexLock lock(&mCondition); mClosed = true; } - mCondition.notify_all(); + mCondition.broadcast(); } bool GLWorkQueue::isClosed() { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; - std::lock_guard lock(mMutex); + LLMutexLock lock(&mCondition); return mClosed; } diff --git a/indra/llui/code/alsyntaxgrammar.cpp b/indra/llui/code/alsyntaxgrammar.cpp index 3b07f86b6e..f329e491f2 100644 --- a/indra/llui/code/alsyntaxgrammar.cpp +++ b/indra/llui/code/alsyntaxgrammar.cpp @@ -28,13 +28,15 @@ #include "alregex.h" #include "altextchars.h" +#include "llcoros.h" #include "lldir.h" #include "llsdserialize.h" #include "llstring.h" #include #include -#include + +#include LLCOROS_MUTEX_HEADER // --- kinds ------------------------------------------------------------------- @@ -213,11 +215,12 @@ U32 ALSyntaxWords::tableBit(std::string_view table) { // Tables by name across every grammar and every set of words: a handful // (functions, constants, keywords and the like). Named as grammars load - // and words are taught, which is rare, so a lock costs nothing. - static std::mutex lock; + // and words are taught, which is rare, so a lock costs nothing; and on + // whichever thread, or coroutine, does either. + static LLCoros::Mutex lock; static std::vector names; - const std::lock_guard held(lock); - const auto it = std::find(names.begin(), names.end(), table); + const LLCoros::LockType held(lock); + const auto it = std::find(names.begin(), names.end(), table); if (it != names.end()) { return static_cast(it - names.begin()); diff --git a/indra/llui/text/altextfind.cpp b/indra/llui/text/altextfind.cpp index 9d88a2da18..9329ecf376 100644 --- a/indra/llui/text/altextfind.cpp +++ b/indra/llui/text/altextfind.cpp @@ -28,15 +28,17 @@ #include "altextfind.h" #include "alserialworker.h" +#include "llcoros.h" #include "llsingleton.h" #include -#include #include #include -#include #include +#include LLCOROS_MUTEX_HEADER +#include LLCOROS_CONDVAR_HEADER + namespace { // The one thread every find's long searches take turns on: made with @@ -78,23 +80,25 @@ namespace }; } +// Between the worker and the main thread, where a coroutine may be what lets +// a search go or collects it. struct ALTextFind::Working { - std::mutex lock; - std::condition_variable done; - bool finished = false; - std::vector found; - std::string error; + LLCoros::Mutex lock; + LLCoros::ConditionVariable done; + bool finished = false; + std::vector found; + std::string error; // The text to look through, until the worker takes it: let go of at // once where the search is, rather than when its turn comes. - std::string text; + std::string text; // Set where nobody takes what it finds: a search not begun is passed // over, and one under way stops at its next match or line. - std::atomic dropped{ false }; + std::atomic dropped{ false }; // What it was asked, to ask it again where the text has moved on. - U32 version = 0; - std::string query; - ALTextSearchOptions options; + U32 version = 0; + std::string query; + ALTextSearchOptions options; }; ALTextFind::~ALTextFind() @@ -120,7 +124,7 @@ void ALTextFind::letGo() mWorking->dropped = true; std::string text; { - std::lock_guard guard(mWorking->lock); + LLCoros::LockType guard(mWorking->lock); text.swap(mWorking->text); } mWorking.reset(); @@ -168,7 +172,7 @@ void ALTextFind::search(const ALTextDocument& doc, const std::string& query, con const bool posted = ALTextFindThread::instance().post([working, query, asked, scope]() { std::string text; { - std::lock_guard guard(working->lock); + LLCoros::LockType guard(working->lock); text.swap(working->text); } if (working->dropped) @@ -190,7 +194,7 @@ void ALTextFind::search(const ALTextDocument& doc, const std::string& query, con found.clear(); error = fault.what(); } - std::lock_guard guard(working->lock); + LLCoros::LockType guard(working->lock); working->found = std::move(found); working->error = std::move(error); working->finished = true; @@ -232,8 +236,8 @@ bool ALTextFind::collect(const ALTextDocument& doc, const ALTextRange& selection std::shared_ptr working = mWorking; // A worker closing passes over what waits for it, which then never // comes: not waited for, but looked through here. - const bool closing = wait && (ALTextFindThread::wasDeleted() || ALTextFindThread::instance().closing()); - std::unique_lock guard(working->lock); + const bool closing = wait && (ALTextFindThread::wasDeleted() || ALTextFindThread::instance().closing()); + LLCoros::LockType guard(working->lock); if (closing && !working->finished) { const std::string query = working->query; diff --git a/indra/llui/text/altextsearch.cpp b/indra/llui/text/altextsearch.cpp index 4aaa660fe2..a0fd49ac41 100644 --- a/indra/llui/text/altextsearch.cpp +++ b/indra/llui/text/altextsearch.cpp @@ -27,6 +27,7 @@ #include "altextsearch.h" #include "altextchars.h" +#include "llcoros.h" #include @@ -35,11 +36,12 @@ #include #include #include -#include #include #include #include +#include LLCOROS_MUTEX_HEADER + namespace { bool wholeWord(const std::string& line, S32 begin, S32 end) @@ -205,8 +207,9 @@ namespace // replace-all asks for the same one once per match, a find bar once a // key, and a whole-word vim pattern asks for two in turn, which one // kept alone recompiled both every time. Behind a lock, since a search - // over many scripts may run off the main thread; each handed out - // shared, so that one being used outlives its place here. + // over many scripts may run off the main thread, and on it a coroutine + // may ask; each handed out shared, so that one being used outlives its + // place here. std::shared_ptr compiledOnce(std::string_view query, const ALTextSearchOptions& options, std::string* error) { struct Kept @@ -217,9 +220,9 @@ namespace std::string error; }; constexpr size_t KEPT = 4; - static std::mutex lock; + static LLCoros::Mutex lock; static std::vector kept; - std::lock_guard guard(lock); + LLCoros::LockType guard(lock); auto found = std::find_if(kept.begin(), kept.end(), [&](const Kept& one) { return one.caseSensitive == options.caseSensitive && one.query == query; }); if (found == kept.end()) { diff --git a/indra/lscript/allslreference.cpp b/indra/lscript/allslreference.cpp index 88712cd841..1a20defed7 100644 --- a/indra/lscript/allslreference.cpp +++ b/indra/lscript/allslreference.cpp @@ -27,6 +27,7 @@ #include "allslreference.h" #include "llapp.h" +#include "llmutex.h" #include "lscript_byteformat.h" #include "lscript_rt_interface.h" @@ -43,7 +44,6 @@ #include #include #include -#include #include namespace @@ -129,8 +129,8 @@ namespace ALLSLReference::Result ALLSLReference::compile(const std::string& text, Target target) { // LL's compiler keeps the parse, its scopes and its errors in globals. - static std::mutex one_at_a_time; - std::lock_guard lock(one_at_a_time); + static LLMutex one_at_a_time; + LLMutexLock lock(&one_at_a_time); // A folder of this compile's own for the files LL's compiler reads and // writes, gone however the compile ends: an LL_ERRS inside it is thrown diff --git a/indra/newview/alscriptanalysis.cpp b/indra/newview/alscriptanalysis.cpp index abeaecbace..643ca0af3c 100644 --- a/indra/newview/alscriptanalysis.cpp +++ b/indra/newview/alscriptanalysis.cpp @@ -33,14 +33,16 @@ #include "llviewercontrol.h" #include "llsyntaxid.h" #include "alsaid.h" +#include "llcoros.h" #include "lltrans.h" #include #include -#include #include +#include LLCOROS_MUTEX_HEADER + namespace { // An answer to whoever asked, on the main thread, in the viewer's @@ -202,12 +204,13 @@ std::string alScriptKeyedWordsAnyThread(const std::string& key, const std::vecto // reads them from are parsed once, at startup, and only read after, // so its own lookup is safe from any thread; what it finds kept here, // under a lock, for the several hundred notes a conversion may write. - static std::mutex lock; + // Any thread, a coroutine on the main one too. + static LLCoros::Mutex lock; static boost::unordered_flat_map, ll::string_hash, std::equal_to<>> found; std::optional text; { - const std::lock_guard held(lock); - auto it = found.find(key); + const LLCoros::LockType held(lock); + auto it = found.find(key); if (it == found.end()) { std::string skin; diff --git a/indra/newview/alscriptanalysislane.cpp b/indra/newview/alscriptanalysislane.cpp index cb29e07ea1..a930ed8f9d 100644 --- a/indra/newview/alscriptanalysislane.cpp +++ b/indra/newview/alscriptanalysislane.cpp @@ -74,7 +74,7 @@ ALScriptAnalysisLane::ALScriptAnalysisLane(std::string name, std::function(std::move(name), [this]() { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); ALLuauService::cancel(mRunningStop); }); } @@ -89,7 +89,7 @@ bool ALScriptAnalysisLane::post(const std::string& key, U8 rank, bool yields, Jo const std::string id = job.request.id; const U32 version = job.request.version; { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); if (mQueue.add(key, id, version, rank, std::move(job), yields)) { // What runs is answering something no longer wanted, or gives @@ -99,7 +99,7 @@ bool ALScriptAnalysisLane::post(const std::string& key, U8 rank, bool yields, Jo } if (!mThread->post([this]() { runNext(); })) { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); mQueue.forget(id); return false; } @@ -108,7 +108,7 @@ bool ALScriptAnalysisLane::post(const std::string& key, U8 rank, bool yields, Jo void ALScriptAnalysisLane::forget(const std::string& id) { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); if (mQueue.forget(id)) { // What runs for it answers nobody now, nor waits again. @@ -134,7 +134,7 @@ void ALScriptAnalysisLane::runNext() // What stops it, where it is a question. ALLuauService::Stop stop; { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); next = mQueue.take(); if (!next) { @@ -161,7 +161,7 @@ void ALScriptAnalysisLane::runNext() LL_WARNS("ScriptAnalysis") << "Engine work failed: " << e.what() << LL_ENDL; } { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); mQueue.finished(); } mMain.run(job.engineDone); @@ -177,7 +177,7 @@ void ALScriptAnalysisLane::runNext() bool unwanted = false; bool again = false; { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); unwanted = mQueue.superseded(); // Stopped as it gave way to the front tab, and wanted still: it waits // again, behind what it gave way to, and answers after -- put back @@ -220,7 +220,7 @@ void ALScriptAnalysisLane::runNext() warm.callback = nullptr; warm.request.kind = Kind::Warm; { - const std::lock_guard lock(mQueueMutex); + const LLCoros::LockType lock(mQueueMutex); mQueue.add(keyOf(warm.request), asked.id, asked.version, rankOf(warm.request), std::move(warm)); } mThread->post([this]() { runNext(); }); diff --git a/indra/newview/alscriptanalysislane.h b/indra/newview/alscriptanalysislane.h index 067cf1676d..f615ce87f5 100644 --- a/indra/newview/alscriptanalysislane.h +++ b/indra/newview/alscriptanalysislane.h @@ -26,12 +26,14 @@ #include "alscriptanalysis.h" #include "alscriptjobqueue.h" +#include "llcoros.h" #include #include -#include #include +#include LLCOROS_MUTEX_HEADER + class ALScriptAnalyzer; class ALSerialWorker; @@ -122,9 +124,10 @@ class ALScriptAnalysisLane std::unique_ptr mThread; // Touched only on the thread. std::unique_ptr mAnalyzer; - // What waits, written on the main thread and taken on the thread, and - // the stop of the job running; both under the lock. - std::mutex mQueueMutex; + // What waits, written on the main thread, a coroutine there among what + // writes, and taken on the thread, and the stop of the job running; both + // under the lock. + LLCoros::Mutex mQueueMutex; ALScriptJobQueue mQueue; ALLuauService::Stop mRunningStop; }; diff --git a/indra/newview/llperfstats.cpp b/indra/newview/llperfstats.cpp index 35ce250095..2cc42e2494 100644 --- a/indra/newview/llperfstats.cpp +++ b/indra/newview/llperfstats.cpp @@ -50,7 +50,7 @@ namespace LLPerfStats U32 lastGlobalPrefChange{0}; U32 lastSleepedFrame{0}; U64 meanFrameTime{0}; - std::mutex bufferToggleLock{}; + LLMutex bufferToggleLock; F64 clock_hertz{0.0}; U32 vsync_max_fps{60}; @@ -208,7 +208,7 @@ namespace LLPerfStats // swap the buffers if(enabled()) { - std::lock_guard lock{bufferToggleLock}; + LLMutexLock lock(&bufferToggleLock); writeBuffer ^= 1; }; // note we are relying on atomic updates here. The risk is low and would cause minor errors in the stats display. @@ -263,7 +263,7 @@ namespace LLPerfStats // swap the clean buffers in if(enabled()) { - std::lock_guard lock{bufferToggleLock}; + LLMutexLock lock(&bufferToggleLock); writeBuffer ^= 1; }; // repeat before we start processing new stuff diff --git a/indra/newview/llperfstats.h b/indra/newview/llperfstats.h index 04e4f6ecda..4afef47f09 100644 --- a/indra/newview/llperfstats.h +++ b/indra/newview/llperfstats.h @@ -31,7 +31,7 @@ #include #include #include -#include +#include "llmutex.h" #include "lluuid.h" #include "lltimer.h" #include "llapp.h" @@ -68,7 +68,7 @@ namespace LLPerfStats extern U32 lastGlobalPrefChange; extern U32 lastSleepedFrame; extern U64 meanFrameTime; - extern std::mutex bufferToggleLock; + extern LLMutex bufferToggleLock; enum class ObjType_t{ OT_GENERAL=0, // Also Unknown. Used for n/a type stats such as scenery diff --git a/indra/newview/llskinningutil.cpp b/indra/newview/llskinningutil.cpp index 9f8137d3c0..c770c55bd3 100644 --- a/indra/newview/llskinningutil.cpp +++ b/indra/newview/llskinningutil.cpp @@ -32,13 +32,13 @@ #include "llvoavatar.h" #include "llviewercontrol.h" #include "llmeshrepository.h" +#include "llmutex.h" #include "llvolume.h" #include "llrigginginfo.h" #include #include -#include #define DEBUG_SKINNING LL_DEBUG @@ -254,7 +254,7 @@ namespace boost::unordered_flat_map mByNum; }; - std::mutex sPublishedJointNumsMutex; + LLMutex sPublishedJointNumsMutex; std::shared_ptr sPublishedJointNums; void collect_joint_nums(LLJoint* joint, PublishedJointNums& nums, LLVOAvatar* avatar) @@ -293,7 +293,7 @@ void LLSkinningUtil::publishJointNums(LLVOAvatar* avatar) } } - std::lock_guard lock(sPublishedJointNumsMutex); + LLMutexLock lock(&sPublishedJointNumsMutex); sPublishedJointNums = std::move(nums); } @@ -301,7 +301,7 @@ bool LLSkinningUtil::initJointNumsFromPublished(LLMeshSkinInfo* skin) { std::shared_ptr nums; { - std::lock_guard lock(sPublishedJointNumsMutex); + LLMutexLock lock(&sPublishedJointNumsMutex); nums = sPublishedJointNums; } if (!nums) diff --git a/indra/newview/llvoavatarself.cpp b/indra/newview/llvoavatarself.cpp index 3c55b9a0f9..8830d9d5dc 100644 --- a/indra/newview/llvoavatarself.cpp +++ b/indra/newview/llvoavatarself.cpp @@ -725,7 +725,7 @@ void LLVOAvatarSelf::idleUpdate(LLAgent &agent, const F64 &time) // virtual LLJoint* LLVOAvatarSelf::getJoint(std::string_view name) { - std::lock_guard lock(mJointMapMutex); + LLCoros::LockType lock(mJointMapMutex); LLJoint* jointp = nullptr; jointp = LLVOAvatar::getJoint(name); if (!jointp && mScreenp) @@ -747,7 +747,7 @@ LLJoint* LLVOAvatarSelf::getJoint(std::string_view name) //virtual void LLVOAvatarSelf::renderJoints() { - std::lock_guard lock(mJointMapMutex); + LLCoros::LockType lock(mJointMapMutex); LLVOAvatar::renderJoints(); } diff --git a/indra/newview/llvoavatarself.h b/indra/newview/llvoavatarself.h index 2df1a28406..42aac480a3 100644 --- a/indra/newview/llvoavatarself.h +++ b/indra/newview/llvoavatarself.h @@ -33,6 +33,7 @@ #include #include "lleventcoro.h" #include "llcoros.h" +#include LLCOROS_MUTEX_HEADER struct LocalTextureData; class LLInventoryCallback; @@ -110,7 +111,7 @@ class LLVOAvatarSelf : // weights the avatar did not already have. Read by the profiler. static S32 sParamWeightsChanged; - std::mutex mJointMapMutex; // getJoint gets used from mesh thread + LLCoros::Mutex mJointMapMutex; // getJoint gets used from mesh thread, and from coroutines /******************************************************************************** ** ** diff --git a/indra/newview/tests/alscriptanalysislane_test.cpp b/indra/newview/tests/alscriptanalysislane_test.cpp index 033f4a8a6a..f9f707f18e 100644 --- a/indra/newview/tests/alscriptanalysislane_test.cpp +++ b/indra/newview/tests/alscriptanalysislane_test.cpp @@ -27,15 +27,15 @@ #include "../alscriptanalysislane.h" #include "../alscriptanalyzers.h" +#include "llmutex.h" + #include #include "Luau/Cancellation.h" #include -#include #include #include -#include #include namespace @@ -43,29 +43,27 @@ namespace // What the lanes answered, in the order they did, from their threads. struct Answers { - std::mutex mutex; - std::condition_variable changed; + LLCondition changed; std::vector said; void add(std::string one) { { - const std::lock_guard lock(mutex); + LLMutexLock lock(&changed); said.push_back(std::move(one)); } - changed.notify_all(); + changed.broadcast(); } // Whether `count` answers came within a few seconds. bool waitFor(size_t count) { - std::unique_lock lock(mutex); - return changed.wait_for(lock, std::chrono::seconds(5), [&] { return said.size() >= count; }); + return changed.waitFor(std::chrono::seconds(5), [&] { return said.size() >= count; }); } std::string all() { - const std::lock_guard lock(mutex); + LLMutexLock lock(&changed); std::string out; for (const std::string& one : said) { @@ -83,32 +81,30 @@ namespace public: struct Gate { - std::mutex mutex; - std::condition_variable changed; - bool open = false; + LLCondition changed; + bool open = false; // Held each time it is asked, not the first time alone: a check // that is slow however often it is started over. - bool always = false; - int entered = 0; + bool always = false; + int entered = 0; void release() { { - const std::lock_guard lock(mutex); + LLMutexLock lock(&changed); open = true; } - changed.notify_all(); + changed.broadcast(); } bool waitEntered(int times = 1) { - std::unique_lock lock(mutex); - return changed.wait_for(lock, std::chrono::seconds(5), [&] { return entered >= times; }); + return changed.waitFor(std::chrono::seconds(5), [&] { return entered >= times; }); } int enteredNow() { - const std::lock_guard lock(mutex); + LLMutexLock lock(&changed); return entered; } }; @@ -126,19 +122,21 @@ namespace { return; } - std::unique_lock lock(mGate->mutex); - ++mGate->entered; - mGate->changed.notify_all(); + { + LLMutexLock lock(&mGate->changed); + ++mGate->entered; + } + mGate->changed.broadcast(); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); // Held once, unless always: run again, it answers at once. - while (!mGate->open && (mGate->always || mGate->entered == 1) && std::chrono::steady_clock::now() < deadline) + const auto let_go = [this] { return mGate->open || !(mGate->always || mGate->entered == 1); }; + while (!mGate->changed.waitFor(std::chrono::milliseconds(5), let_go) && std::chrono::steady_clock::now() < deadline) { if (mStops && setup.stop && setup.stop->requested()) { mStopped = true; return; } - mGate->changed.wait_for(lock, std::chrono::milliseconds(5)); } } bool stopped() const override { return mStopped; }