From f51e86a532b972df546c82af04334a25c748aebf Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:12:08 -0300 Subject: [PATCH 01/13] feat(platform): native Windows support (MSVC ABI + Clang, zero external deps) Port ripwire natively to Windows (x64) using Clang with MSVC ABI, preserving zero external runtime dependencies (linking only system kernel32, ws2_32, advapi32, and shell32). MECHANISM & ARCHITECTURE: - Platform shim layer in src/infra/platform_compat.{h,cpp} and minimal POSIX compatibility headers in src/infra/compat/ (sys/socket.h, unistd.h, poll.h, sys/wait.h, etc.) routed via -include / /FI compiler options. - Atomic cache rename via MoveFileExA (MOVEFILE_REPLACE_EXISTING) after closing open file descriptors on Windows. - Win32 Job Object subprocess runner in src/verbs_change.h for isolated child process management, timeout enforcement, and asynchronous stdout capture. - cmd.exe command-line adaptations: double-quoted git format strings to prevent unintended pipe interpretation, and short-path generation (GetShortPathNameA) for stream redirection without quotes (< shortPath). - Socket safety: Winsock automatic initialization and explicit closesocket/CRT handle separation in rw_close. - Application manifest embedded in executables opting into longPathAware (NTFS 32k path lengths) and UTF-8 active code page. - Documentation in CONTRIBUTING.md and automated validation in ci.yml. GATE & VALIDATIONS (Windows 11 x64): - Doctor: ripwire.exe . --doctor -> 7/7 checks passed. - Determinism: test/det-gate.sh passed (baseline + nesting-kind + width arm at 631 B). - Retrieval accuracy: --eval-retrieval -> MRR 0.967 / recall@10 99.4% (3,038 symbols). - Visualization: --html generates valid self-contained interactive force-directed graph. - MCP Server: HTTP 2024-11-05 endpoint serves 31 tools and processes queries. --- .github/workflows/ci.yml | 32 +++ CMakeLists.txt | 31 ++- CONTRIBUTING.md | 23 ++ cmake/PortableFlags.cmake | 5 +- src/crossref.h | 9 + src/gitoracle.h | 4 + src/infra/compat/arpa/inet.h | 7 + src/infra/compat/netinet/in.h | 7 + src/infra/compat/netinet/tcp.h | 7 + src/infra/compat/poll.h | 14 + src/infra/compat/sys/file.h | 2 + src/infra/compat/sys/socket.h | 7 + src/infra/compat/sys/time.h | 6 + src/infra/compat/sys/wait.h | 18 ++ src/infra/compat/unistd.h | 25 ++ src/infra/jsonesc.h | 14 + src/infra/platform_compat.cpp | 452 +++++++++++++++++++++++++++++++ src/infra/platform_compat.h | 283 +++++++++++++++++++ src/infra/profileScope.h | 2 + src/infra/win32/ripwire.manifest | 9 + src/ingest_cache.h | 12 + src/mcpserver.h | 10 +- src/quality.h | 32 +++ src/renamemine.h | 4 + src/verbs_change.h | 131 +++++++++ src/verbs_doctor.h | 4 + 26 files changed, 1147 insertions(+), 3 deletions(-) create mode 100644 src/infra/compat/arpa/inet.h create mode 100644 src/infra/compat/netinet/in.h create mode 100644 src/infra/compat/netinet/tcp.h create mode 100644 src/infra/compat/poll.h create mode 100644 src/infra/compat/sys/file.h create mode 100644 src/infra/compat/sys/socket.h create mode 100644 src/infra/compat/sys/time.h create mode 100644 src/infra/compat/sys/wait.h create mode 100644 src/infra/compat/unistd.h create mode 100644 src/infra/platform_compat.cpp create mode 100644 src/infra/platform_compat.h create mode 100644 src/infra/win32/ripwire.manifest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 523013684..ccd4176e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,6 +289,38 @@ jobs: - name: G4 — xmllint --noout run: ./build/ripwire test/fixture --no-cache | xmllint --noout - + # ─── windows: native Windows validation with MSVC ABI + Clang ──────────────────────────────────────── + windows: + name: windows (windows-latest, clang) + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure (portable — clang + ninja) + run: cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ + + - name: Build + run: cmake --build build -j --target ripwire + + - name: Doctor check + run: .\build\ripwire.exe . --doctor + + - name: Self-run on the fixture + run: .\build\ripwire.exe test/fixture --no-cache + + - name: det-gate — 2-run byte-identical diff + shell: bash + run: | + ./build/ripwire.exe test/fixture --no-cache > run_a.xml + ./build/ripwire.exe test/fixture --no-cache > run_b.xml + diff -q run_a.xml run_b.xml + + - name: Determinism gate (test/det-gate.sh) + shell: bash + run: bash test/det-gate.sh build/ripwire.exe + asan: name: asan (${{ matrix.os }}) strategy: diff --git a/CMakeLists.txt b/CMakeLists.txt index 12130424a..91c63e4dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,10 @@ option(RIPWIRE_NATIVE "build with -march=native (DEV MACHINES ONLY — bakes in include(cmake/PortableFlags.cmake) add_compile_options(${RIPWIRE_ARCH_FLAGS}) +if(WIN32) + add_compile_definitions(_CRT_SECURE_NO_WARNINGS _CRT_NONSTDC_NO_DEPRECATE) +endif() + # ---- link-time optimization: -DRIPWIRE_LTO=ON ---- # The one change the optimization-remarks pass actually justified (docs/OPTREMARKS.md, finding F1). # @@ -493,11 +497,16 @@ set(RIPWIRE_SRCS src/ingest.cpp src/pagerank.cpp src/infra/diagnostics.cpp + src/infra/platform_compat.cpp ) # The global profile uses -ffast-math, but PageRank reductions must not reassociate (the determinism # contract's no-reassociation rule — docs/ARCHITECTURE.md §3). -set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "-fno-fast-math") +if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "/fp:precise") +else() + set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "-fno-fast-math") +endif() # All grammar OBJECT files, gathered once so every target links the same set. set(RIPWIRE_TS_OBJECTS @@ -539,9 +548,19 @@ add_executable(ripwire_probe ${RIPWIRE_SRCS} ${RIPWIRE_TS_OBJECTS}) target_link_libraries(ripwire_probe PRIVATE tree-sitter Threads::Threads) +if(WIN32) + target_link_libraries(ripwire_probe PRIVATE ws2_32) + target_sources(ripwire_probe PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/win32/ripwire.manifest") + if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(ripwire_probe PRIVATE /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + else() + target_compile_options(ripwire_probe PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + endif() +endif() target_include_directories(ripwire_probe PRIVATE ${tree_sitter_SOURCE_DIR}/lib/include ${_ripwire_generated_dir} + $<$:${CMAKE_CURRENT_SOURCE_DIR}/src/infra/compat> src/infra third_party src) @@ -554,9 +573,19 @@ add_executable(ripwire ${RIPWIRE_SRCS} ${RIPWIRE_TS_OBJECTS}) target_link_libraries(ripwire PRIVATE tree-sitter Threads::Threads) +if(WIN32) + target_link_libraries(ripwire PRIVATE ws2_32) + target_sources(ripwire PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/win32/ripwire.manifest") + if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(ripwire PRIVATE /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + else() + target_compile_options(ripwire PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + endif() +endif() target_include_directories(ripwire PRIVATE ${tree_sitter_SOURCE_DIR}/lib/include ${_ripwire_generated_dir} + $<$:${CMAKE_CURRENT_SOURCE_DIR}/src/infra/compat> src/infra third_party src) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96ac89b82..a2498b4c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -75,6 +75,29 @@ is compiled out and freshness comes from the per-request stat sweep — that is a degradation, so it is silent and the staleness contract is unchanged. You can build and run that path on a Mac with `cmake -S . -B build-nokqueue -DCMAKE_CXX_FLAGS=-DRIPWIRE_HAS_KQUEUE=0`. +### Building on Windows + +ripwire builds natively on Windows (x64) with Clang and the MSVC ABI, with zero external runtime +dependencies (linking only system `kernel32`, `ws2_32`, `advapi32`, `shell32`). + +From an **x64 Native Tools Command Prompt for Visual Studio** (with `clang` and `ninja` on `PATH`): + +```cmd +cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ +cmake --build build -j +``` + +Or using the Visual Studio generator with Clang-CL: + +```cmd +cmake -S . -B build -T ClangCL +cmake --build build --config Release +``` + +The resulting binaries (`build/ripwire.exe` and `build/ripwire_probe.exe`) embed an application +manifest opting into `longPathAware` (handling arbitrary deep paths up to NTFS 32k limits) and UTF-8 +active code page. + ### Determinism gate Output is a sorted top-K. A sort has no tolerance band, so the contract is byte-identity: diff --git a/cmake/PortableFlags.cmake b/cmake/PortableFlags.cmake index 60f328439..1bfe8298e 100644 --- a/cmake/PortableFlags.cmake +++ b/cmake/PortableFlags.cmake @@ -34,7 +34,10 @@ if(APPLE AND NOT RIPWIRE_PRETEND_LINUX AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm set(RIPWIRE_IS_APPLE_SILICON ON) endif() -if(RIPWIRE_NATIVE) +if(MSVC) + # MSVC compiler flags: fast math, conformant C++ mode, UTF-8 source/exec charset + set(RIPWIRE_ARCH_FLAGS /O2 /fp:fast /permissive- /utf-8) +elseif(RIPWIRE_NATIVE) set(RIPWIRE_ARCH_FLAGS -O3 -march=native -ffast-math -fno-finite-math-only) elseif(RIPWIRE_IS_APPLE_SILICON) set(RIPWIRE_ARCH_FLAGS -O2 -mcpu=apple-m1 -ffast-math -fno-finite-math-only) diff --git a/src/crossref.h b/src/crossref.h index e26261051..50bba1591 100644 --- a/src/crossref.h +++ b/src/crossref.h @@ -490,8 +490,13 @@ inline void streamBlobs( const std::string& root, const std::vector std::fclose( lf ); } +#ifdef _WIN32 + const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + + " cat-file --batch < " + rw_short_path( listPath ) + " 2>/dev/null"; +#else const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " cat-file --batch < " + shSingleQuote( listPath ) + " 2>/dev/null"; +#endif std::FILE* pipe = popen( cmd.c_str(), "r" ); if( !pipe ) { @@ -690,7 +695,11 @@ struct RefInfo inline std::vector enumerateRefs( const std::string& root, std::string_view filter, const std::string& headSha, std::size_t* filterNameHits = nullptr ) { +#ifdef _WIN32 + const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format=\"%(refname:short)|%(objectname)|%(committerdate:short)\" refs/heads 2>/dev/null" ); +#else const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format='%(refname:short)|%(objectname)|%(committerdate:short)' refs/heads 2>/dev/null" ); +#endif std::vector out; for( std::string_view line : splitLines( raw ) ) { diff --git a/src/gitoracle.h b/src/gitoracle.h index db2e25a7a..5e79f6820 100644 --- a/src/gitoracle.h +++ b/src/gitoracle.h @@ -595,7 +595,11 @@ inline HistoryIndex runProbe( const std::string& root ) const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " log --no-merges --no-color --no-ext-diff --no-textconv --no-renames" +#ifdef _WIN32 + " --format=\"%x01%H %cs\" -p -U0 2>/dev/null"; +#else " --format='%x01%H %cs' -p -U0 2>/dev/null"; +#endif RemovalSite site; const PatchWalk walk = walkGitPatch( cmd, diff --git a/src/infra/compat/arpa/inet.h b/src/infra/compat/arpa/inet.h new file mode 100644 index 000000000..3fa5f63b3 --- /dev/null +++ b/src/infra/compat/arpa/inet.h @@ -0,0 +1,7 @@ +#pragma once +#if defined(_WIN32) +#include +#include +#else +#include_next +#endif diff --git a/src/infra/compat/netinet/in.h b/src/infra/compat/netinet/in.h new file mode 100644 index 000000000..696fe4faf --- /dev/null +++ b/src/infra/compat/netinet/in.h @@ -0,0 +1,7 @@ +#pragma once +#if defined(_WIN32) +#include +#include +#else +#include_next +#endif diff --git a/src/infra/compat/netinet/tcp.h b/src/infra/compat/netinet/tcp.h new file mode 100644 index 000000000..a86fab25b --- /dev/null +++ b/src/infra/compat/netinet/tcp.h @@ -0,0 +1,7 @@ +#pragma once +#if defined(_WIN32) +#include +#include +#else +#include_next +#endif diff --git a/src/infra/compat/poll.h b/src/infra/compat/poll.h new file mode 100644 index 000000000..c1a919b7c --- /dev/null +++ b/src/infra/compat/poll.h @@ -0,0 +1,14 @@ +#pragma once +#include "../platform_compat.h" + + + +#ifdef __cplusplus +namespace rw::compat +{ + int rw_poll( struct pollfd* fds, unsigned long nfds, int timeout ); +} +#ifndef poll + #define poll rw::compat::rw_poll +#endif +#endif diff --git a/src/infra/compat/sys/file.h b/src/infra/compat/sys/file.h new file mode 100644 index 000000000..5e8e62071 --- /dev/null +++ b/src/infra/compat/sys/file.h @@ -0,0 +1,2 @@ +#pragma once +#include "../../platform_compat.h" diff --git a/src/infra/compat/sys/socket.h b/src/infra/compat/sys/socket.h new file mode 100644 index 000000000..119c1989b --- /dev/null +++ b/src/infra/compat/sys/socket.h @@ -0,0 +1,7 @@ +#pragma once +#if defined(_WIN32) +#include +#include +#else +#include_next +#endif diff --git a/src/infra/compat/sys/time.h b/src/infra/compat/sys/time.h new file mode 100644 index 000000000..a15c1b208 --- /dev/null +++ b/src/infra/compat/sys/time.h @@ -0,0 +1,6 @@ +#pragma once +#include +#if defined(_WIN32) +#include +#endif + diff --git a/src/infra/compat/sys/wait.h b/src/infra/compat/sys/wait.h new file mode 100644 index 000000000..a3bcf06af --- /dev/null +++ b/src/infra/compat/sys/wait.h @@ -0,0 +1,18 @@ +#pragma once +#include "../../platform_compat.h" + +#ifndef WIFEXITED + #define WIFEXITED(status) (((status) & 0x7f) == 0) +#endif +#ifndef WEXITSTATUS + #define WEXITSTATUS(status) (((status) >> 8) & 0xff) +#endif +#ifndef WIFSIGNALED + #define WIFSIGNALED(status) (((status) & 0x7f) != 0) +#endif +#ifndef WTERMSIG + #define WTERMSIG(status) ((status) & 0x7f) +#endif +#ifndef WNOHANG + #define WNOHANG 1 +#endif diff --git a/src/infra/compat/unistd.h b/src/infra/compat/unistd.h new file mode 100644 index 000000000..19027f5e6 --- /dev/null +++ b/src/infra/compat/unistd.h @@ -0,0 +1,25 @@ +#pragma once +#include "../platform_compat.h" + +#ifndef STDIN_FILENO + #define STDIN_FILENO 0 +#endif +#ifndef STDOUT_FILENO + #define STDOUT_FILENO 1 +#endif +#ifndef STDERR_FILENO + #define STDERR_FILENO 2 +#endif + +#ifndef F_OK + #define F_OK 0 +#endif +#ifndef X_OK + #define X_OK 1 +#endif +#ifndef W_OK + #define W_OK 2 +#endif +#ifndef R_OK + #define R_OK 4 +#endif diff --git a/src/infra/jsonesc.h b/src/infra/jsonesc.h index 49f48817e..1427f9b49 100644 --- a/src/infra/jsonesc.h +++ b/src/infra/jsonesc.h @@ -267,6 +267,19 @@ inline bool isJsonWs( char c ) noexcept // forwards here instead of carrying its own copy. inline std::string shSingleQuote( const std::string& s ) { +#if defined(_WIN32) + std::string out = "\""; + for( char c : s ) + { + if( c == '"' ) { out += "\\\""; } + else + { + out += c; + } + } + out += "\""; + return out; +#else std::string out = "'"; for( char c : s ) { @@ -278,6 +291,7 @@ inline std::string shSingleQuote( const std::string& s ) } out += "'"; return out; +#endif } } // namespace rw diff --git a/src/infra/platform_compat.cpp b/src/infra/platform_compat.cpp new file mode 100644 index 000000000..9c6eca5b5 --- /dev/null +++ b/src/infra/platform_compat.cpp @@ -0,0 +1,452 @@ +#include "platform_compat.h" + +#if defined(_WIN32) || defined(_MSC_VER) + +#include +#include +#include +#include +#include + +#ifdef fclose + #undef fclose +#endif +#ifdef fflush + #undef fflush +#endif +#ifdef open_memstream + #undef open_memstream +#endif +#ifdef close + #undef close +#endif + +namespace rw::compat +{ + +int rw_flock( int fd, int operation ) noexcept +{ + HANDLE hFile = reinterpret_cast( _get_osfhandle( fd ) ); + if( hFile == INVALID_HANDLE_VALUE ) + { + errno = EBADF; + return -1; + } + + if( operation & LOCK_UN ) + { + OVERLAPPED ov{}; + const BOOL ok = UnlockFileEx( hFile, 0, MAXDWORD, MAXDWORD, &ov ); + if( !ok ) + { + errno = EINVAL; + return -1; + } + return 0; + } + + DWORD flags = 0; + if( operation & LOCK_NB ) + { + flags |= LOCKFILE_FAIL_IMMEDIATELY; + } + if( operation & LOCK_EX ) + { + flags |= LOCKFILE_EXCLUSIVE_LOCK; + } + + OVERLAPPED ov{}; + const BOOL ok = LockFileEx( hFile, flags, 0, MAXDWORD, MAXDWORD, &ov ); + if( !ok ) + { + const DWORD err = GetLastError(); + if( err == ERROR_LOCK_VIOLATION ) + { + errno = EWOULDBLOCK; + } + else + { + errno = EACCES; + } + return -1; + } + return 0; +} + +ssize_t rw_pread( int fd, void* buf, std::size_t count, std::uint64_t offset ) noexcept +{ + HANDLE hFile = reinterpret_cast( _get_osfhandle( fd ) ); + if( hFile == INVALID_HANDLE_VALUE ) + { + errno = EBADF; + return -1; + } + + OVERLAPPED ov{}; + ov.Offset = static_cast( offset & 0xFFFFFFFFull ); + ov.OffsetHigh = static_cast( ( offset >> 32 ) & 0xFFFFFFFFull ); + + DWORD bytesRead = 0; + const BOOL ok = ReadFile( hFile, buf, static_cast( count ), &bytesRead, &ov ); + if( !ok ) + { + const DWORD err = GetLastError(); + if( err == ERROR_HANDLE_EOF ) + { + return 0; + } + errno = EIO; + return -1; + } + return static_cast( bytesRead ); +} + +char* rw_realpath( const char* path, char* resolved_path ) noexcept +{ + if( path == nullptr ) + { + errno = EINVAL; + return nullptr; + } + return _fullpath( resolved_path, path, PATH_MAX ); +} + +std::FILE* rw_popen( const char* command, const char* mode ) +{ + if( command == nullptr || mode == nullptr ) + { + return nullptr; + } + + std::string cmd = command; + // Replace POSIX /dev/null redirection with Windows NUL + const std::string devNull = "/dev/null"; + std::size_t pos = 0; + while( ( pos = cmd.find( devNull, pos ) ) != std::string::npos ) + { + cmd.replace( pos, devNull.length(), "NUL" ); + pos += 3; + } + + // Windows popen in text mode ("r") translates CRLF -> LF and truncates at ^Z (0x1A), + // corrupting binary streams and git cat-file batches. Force binary mode to match POSIX. + std::string winMode = mode; + if( winMode == "r" ) + { + winMode = "rb"; + } + else if( winMode == "w" ) + { + winMode = "wb"; + } + + return _popen( cmd.c_str(), winMode.c_str() ); +} + +std::string rw_short_path( const std::string& path ) +{ + if( path.empty() ) + { + return path; + } + std::string winPath = path; + for( char& c : winPath ) + { + if( c == '/' ) + { + c = '\\'; + } + } + char buf[MAX_PATH]; + const DWORD len = GetShortPathNameA( winPath.c_str(), buf, MAX_PATH ); + if( len > 0 && len < MAX_PATH ) + { + return std::string( buf, len ); + } + return winPath; +} + +int rw_pclose( std::FILE* stream ) +{ + if( stream == nullptr ) + { + return -1; + } + return _pclose( stream ); +} + +std::string rw_self_exe_path() +{ + char buf[MAX_PATH]; + const DWORD len = GetModuleFileNameA( nullptr, buf, MAX_PATH ); + if( len > 0 && len < MAX_PATH ) + { + return std::string( buf, len ); + } + return {}; +} + +struct pollfd; +int rw_poll( struct pollfd* fds, unsigned long nfds, int timeout ) +{ + if( fds == nullptr || nfds == 0 ) + { + if( timeout > 0 ) + { + Sleep( static_cast( timeout ) ); + } + return 0; + } + + struct WinPollFd + { + int fd; + short events; + short revents; + }; + auto* pfds = reinterpret_cast( fds ); + + const DWORD start = GetTickCount(); + for( ;; ) + { + int ready = 0; + for( unsigned long i = 0; i < nfds; ++i ) + { + pfds[i].revents = 0; + HANDLE h = reinterpret_cast( _get_osfhandle( pfds[i].fd ) ); + if( h == INVALID_HANDLE_VALUE ) + { + pfds[i].revents = 0x0020; // POLLNVAL + ready++; + continue; + } + DWORD bytesAvail = 0; + if( PeekNamedPipe( h, nullptr, 0, nullptr, &bytesAvail, nullptr ) ) + { + if( bytesAvail > 0 ) + { + pfds[i].revents |= 0x0001; // POLLIN + ready++; + } + } + else + { + const DWORD err = GetLastError(); + if( err == ERROR_BROKEN_PIPE || err == ERROR_HANDLE_EOF ) + { + pfds[i].revents |= 0x0010; // POLLHUP + ready++; + } + } + } + if( ready > 0 ) + { + return ready; + } + if( timeout >= 0 ) + { + const DWORD elapsed = GetTickCount() - start; + if( elapsed >= static_cast( timeout ) ) + { + return 0; + } + } + Sleep( 5 ); + } +} + +struct MemStreamInfo +{ + char** bufloc; + std::size_t* sizeloc; +}; + +static std::mutex s_memstream_mutex; +static std::unordered_map s_memstreams; + +std::FILE* rw_open_memstream( char** bufloc, std::size_t* sizeloc ) +{ + if( bufloc == nullptr || sizeloc == nullptr ) + { + errno = EINVAL; + return nullptr; + } + + char tempPath[MAX_PATH]; + if( GetTempPathA( MAX_PATH, tempPath ) == 0 ) + { + return nullptr; + } + + char tempFileName[MAX_PATH]; + if( GetTempFileNameA( tempPath, "rwm", 0, tempFileName ) == 0 ) + { + return nullptr; + } + + HANDLE hFile = CreateFileA( + tempFileName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE, + nullptr + ); + if( hFile == INVALID_HANDLE_VALUE ) + { + return nullptr; + } + + int fd = _open_osfhandle( reinterpret_cast( hFile ), _O_RDWR | _O_BINARY ); + if( fd == -1 ) + { + CloseHandle( hFile ); + return nullptr; + } + + std::FILE* fp = _fdopen( fd, "w+b" ); + if( !fp ) + { + _close( fd ); + return nullptr; + } + + *bufloc = static_cast( std::malloc( 1 ) ); + if( *bufloc ) + { + ( *bufloc )[0] = '\0'; + } + *sizeloc = 0; + + std::lock_guard lock( s_memstream_mutex ); + s_memstreams[fp] = MemStreamInfo{ bufloc, sizeloc }; + return fp; +} + +int rw_fflush( std::FILE* stream ) +{ + if( stream == nullptr ) + { + return 0; + } + + std::lock_guard lock( s_memstream_mutex ); + auto it = s_memstreams.find( stream ); + if( it != s_memstreams.end() ) + { + auto& info = it->second; + ::fflush( stream ); + long currentPos = std::ftell( stream ); + std::fseek( stream, 0, SEEK_END ); + long len = std::ftell( stream ); + if( len < 0 ) + { + len = 0; + } + std::fseek( stream, 0, SEEK_SET ); + + char* newBuf = static_cast( std::realloc( *info.bufloc, len + 1 ) ); + if( newBuf ) + { + size_t readBytes = 0; + if( len > 0 ) + { + readBytes = std::fread( newBuf, 1, len, stream ); + } + newBuf[readBytes] = '\0'; + *info.bufloc = newBuf; + *info.sizeloc = readBytes; + } + std::fseek( stream, currentPos, SEEK_SET ); + return 0; + } + return ::fflush( stream ); +} + +int rw_fclose( std::FILE* stream ) +{ + if( stream == nullptr ) + { + return 0; + } + + MemStreamInfo info{}; + bool isMem = false; + { + std::lock_guard lock( s_memstream_mutex ); + auto it = s_memstreams.find( stream ); + if( it != s_memstreams.end() ) + { + info = it->second; + isMem = true; + s_memstreams.erase( it ); + } + } + + if( isMem ) + { + ::fflush( stream ); + std::fseek( stream, 0, SEEK_END ); + long len = std::ftell( stream ); + if( len < 0 ) + { + len = 0; + } + std::fseek( stream, 0, SEEK_SET ); + + char* newBuf = static_cast( std::realloc( *info.bufloc, len + 1 ) ); + if( newBuf ) + { + size_t readBytes = 0; + if( len > 0 ) + { + readBytes = std::fread( newBuf, 1, len, stream ); + } + newBuf[readBytes] = '\0'; + *info.bufloc = newBuf; + *info.sizeloc = readBytes; + } + return ::fclose( stream ); + } + + return ::fclose( stream ); +} + +int rw_close( int fd ) +{ + if( fd < 0 ) + { + return -1; + } + if( closesocket( static_cast( fd ) ) == 0 ) + { + return 0; + } + if( WSAGetLastError() == WSAENOTSOCK ) + { + if( _get_osfhandle( fd ) != -1 ) + { + return _close( fd ); + } + } + return -1; +} + +struct WinsockAutoInit +{ + WinsockAutoInit() + { + WSADATA d; + WSAStartup( MAKEWORD( 2, 2 ), &d ); + } + ~WinsockAutoInit() + { + WSACleanup(); + } +}; +static WinsockAutoInit s_winsockAutoInit; + +} // namespace rw::compat + +#endif diff --git a/src/infra/platform_compat.h b/src/infra/platform_compat.h new file mode 100644 index 000000000..9ec0f4635 --- /dev/null +++ b/src/infra/platform_compat.h @@ -0,0 +1,283 @@ +#pragma once + +#if defined(_WIN32) || defined(_MSC_VER) + + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + #ifndef _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_NONSTDC_NO_DEPRECATE + #endif + + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #ifndef NOMINMAX + #define NOMINMAX + #endif + + #include + #include + #include + #ifdef near + #undef near + #endif + #ifdef far + #undef far + #endif + + #include + #include + #include + #include + #include + #include + #include + #include + #include + + #ifndef PATH_MAX + #define PATH_MAX 4096 + #endif + + #ifndef O_CLOEXEC + #ifdef _O_NOINHERIT + #define O_CLOEXEC _O_NOINHERIT + #else + #define O_CLOEXEC 0 + #endif + #endif + + #ifndef S_ISREG + #define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) + #endif + #ifndef S_ISDIR + #define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) + #endif + #ifndef S_ISLNK + #define S_ISLNK(m) 0 + #endif + + #ifndef LOCK_SH + #define LOCK_SH 1 + #define LOCK_EX 2 + #define LOCK_NB 4 + #define LOCK_UN 8 + #endif + +#ifdef __cplusplus + + #include + #include + #include + + using ssize_t = SSIZE_T; + + namespace rw::compat + { + int rw_flock( int fd, int operation ) noexcept; + ssize_t rw_pread( int fd, void* buf, std::size_t count, std::uint64_t offset ) noexcept; + char* rw_realpath( const char* path, char* resolved_path ) noexcept; + std::FILE* rw_popen( const char* command, const char* mode ); + int rw_pclose( std::FILE* stream ); + std::string rw_self_exe_path(); + std::FILE* rw_open_memstream( char** bufloc, std::size_t* sizeloc ); + int rw_fclose( std::FILE* stream ); + int rw_fflush( std::FILE* stream ); + int rw_close( int fd ); + std::string rw_short_path( const std::string& path ); + inline int rw_rename( const char* oldname, const char* newname ) noexcept + { + for( int attempt = 0; attempt < 8; ++attempt ) + { + if( MoveFileExA( oldname, newname, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED ) ) + { + return 0; + } + const DWORD err = GetLastError(); + if( err != ERROR_ACCESS_DENIED && err != ERROR_SHARING_VIOLATION ) + { + break; + } + Sleep( 5 ); + } + return -1; + } + } + + namespace std + { + using rw::compat::rw_fclose; + using rw::compat::rw_fflush; + using rw::compat::rw_rename; + } + + using rw::compat::rw_fclose; + using rw::compat::rw_fflush; + using rw::compat::rw_close; + using rw::compat::rw_rename; + using rw::compat::rw_short_path; + + #ifndef rename + #define rename rw_rename + #endif + + // Transparent polyfills for POSIX symbols called as ::popen, ::pread, etc. + #ifndef popen + #define popen rw::compat::rw_popen + #endif + #ifndef pclose + #define pclose rw::compat::rw_pclose + #endif + #ifndef pread + #define pread rw::compat::rw_pread + #endif + #ifndef realpath + #define realpath rw::compat::rw_realpath + #endif + #ifndef flock + #define flock rw::compat::rw_flock + #endif + #ifndef open_memstream + #define open_memstream rw::compat::rw_open_memstream + #endif + #ifndef fclose + #define fclose rw_fclose + #endif + #ifndef fflush + #define fflush rw_fflush + #endif + inline int close( int fd ) + { + return rw::compat::rw_close( fd ); + } + + #include + inline struct tm* rw_localtime_r( const time_t* timer, struct tm* buf ) noexcept + { + return localtime_s( buf, timer ) == 0 ? buf : nullptr; + } + inline struct tm* rw_gmtime_r( const time_t* timer, struct tm* buf ) noexcept + { + return gmtime_s( buf, timer ) == 0 ? buf : nullptr; + } + #ifndef localtime_r + #define localtime_r rw_localtime_r + #endif + #ifndef gmtime_r + #define gmtime_r rw_gmtime_r + #endif + + inline int mkdir( const char* path, int /*mode*/ ) + { + return _mkdir( path ); + } + + inline int lstat( const char* path, struct stat* buf ) + { + return ::stat( path, buf ); + } + + inline unsigned int getuid() noexcept + { + return 1000; + } + + inline int nanosleep( const struct timespec* req, struct timespec* /*rem*/ ) noexcept + { + if( req ) + { + DWORD ms = static_cast( req->tv_sec * 1000 + ( req->tv_nsec + 999999 ) / 1000000 ); + Sleep( ms ); + } + return 0; + } + + inline int fchmod( int /*fd*/, int /*mode*/ ) noexcept + { + return 0; + } + + inline int fsync( int fd ) noexcept + { + return _commit( fd ); + } + + namespace rw::compat + { + inline int rw_setsockopt( int s, int level, int optname, const void* optval, int optlen ) + { + if( level == SOL_SOCKET && optname == SO_RCVTIMEO && optlen == sizeof( timeval ) ) + { + const auto* tv = static_cast( optval ); + DWORD ms = static_cast( tv->tv_sec * 1000 + tv->tv_usec / 1000 ); + return ::setsockopt( static_cast( s ), level, optname, reinterpret_cast( &ms ), sizeof( ms ) ); + } + return ::setsockopt( static_cast( s ), level, optname, static_cast( optval ), optlen ); + } + } + + using rw::compat::rw_setsockopt; + + #ifndef setsockopt + #define setsockopt rw_setsockopt + #endif + + #include + namespace std + { + #if defined(_MSC_VER) && !defined(__cpp_lib_format_ranges) + template + using format_string = _Fmt_string<_Args...>; + #endif + } + +#else + + typedef SSIZE_T ssize_t; + +#endif // __cplusplus + +#else + + #include + #include + #include + #include + + #ifdef __cplusplus + namespace rw::compat + { + inline int rw_flock( int fd, int operation ) noexcept + { + return ::flock( fd, operation ); + } + + inline ssize_t rw_pread( int fd, void* buf, std::size_t count, std::uint64_t offset ) noexcept + { + return ::pread( fd, buf, count, static_cast( offset ) ); + } + + inline char* rw_realpath( const char* path, char* resolved_path ) noexcept + { + return ::realpath( path, resolved_path ); + } + + inline std::FILE* rw_popen( const char* command, const char* mode ) + { + return ::popen( command, mode ); + } + + inline int rw_pclose( std::FILE* stream ) + { + return ::pclose( stream ); + } + + inline std::string rw_self_exe_path() + { + return {}; + } + } + #endif + +#endif diff --git a/src/infra/profileScope.h b/src/infra/profileScope.h index 95d9804c4..0b97ca01b 100644 --- a/src/infra/profileScope.h +++ b/src/infra/profileScope.h @@ -79,7 +79,9 @@ #include #include #include +#if !defined(_WIN32) #include +#endif #include "fastmath.h" // ALWAYS_INLINE + cache-line size (via platform.h), fastmath::min/max (integral) #include "profilePmc.h" // prof::pmc — optional Apple Silicon HW counters diff --git a/src/infra/win32/ripwire.manifest b/src/infra/win32/ripwire.manifest new file mode 100644 index 000000000..4c583dcdd --- /dev/null +++ b/src/infra/win32/ripwire.manifest @@ -0,0 +1,9 @@ + + + + + true + UTF-8 + + + diff --git a/src/ingest_cache.h b/src/ingest_cache.h index bd3a9b049..2f2e7598e 100644 --- a/src/ingest_cache.h +++ b/src/ingest_cache.h @@ -879,6 +879,7 @@ struct ReadFd ReadFd& operator=( const ReadFd& ) = delete; ReadFd( ReadFd&& other ) noexcept : fd( other.fd ) { other.fd = -1; } ~ReadFd() { if( fd >= 0 ) { ::close( fd ); } } + void close() noexcept { if( fd >= 0 ) { ::close( fd ); fd = -1; } } // openOnce, not a move-assignment: the only mutation this type needs is "fill an empty guard", and // a move-assign operator here would be a byte-for-byte clone of ingest_sidecap.h's TreeGuard one @@ -943,6 +944,7 @@ struct CacheFrame long long mtimeNs = -1;// the blob's own mtime — the warm-run racy-rule reference bool ok = false; CacheReject reason = CacheReject::Absent; // meaningful only while ok == false + void close() noexcept { blob.close(); } }; // pread the whole of [ off, off+n ) into `dst`. Short reads are retried (a pread on a regular file can @@ -2138,6 +2140,7 @@ inline void saveCache( const std::string& path, std::string_view rootDir, const PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: offset table + trailer" ); finishCacheBlob( w, table ); } + const_cast( prev ).close(); PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: write + rename" ); // unique per-process temp so two concurrent runs (this repo runs ~20 parallel sessions) don't @@ -2165,12 +2168,21 @@ inline void saveCache( const std::string& path, std::string_view rootDir, const DEGRADED_PATH_ALERT( "ingest: saveCache write failed (short write or fclose error) — old cache preserved" ); return; } +#if defined(_WIN32) + if( !MoveFileExA( tmp.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED ) ) + { + std::remove( tmp.c_str() ); + DEGRADED_PATH_ALERT( "ingest: saveCache rename(tmp -> cache) failed — old cache preserved" ); + return; + } +#else if( std::rename( tmp.c_str(), path.c_str() ) != 0 ) { std::remove( tmp.c_str() ); // clean up on failure DEGRADED_PATH_ALERT( "ingest: saveCache rename(tmp -> cache) failed — old cache preserved" ); return; } +#endif // A5 (cache-dir hygiene): --doctor measured ~11,914 ripwire-* blobs / 2.4 GB accumulating in the cache-ladder // dir because only the qsnap/qheadsnap families ever evicted — this main parse-cache family (this very diff --git a/src/mcpserver.h b/src/mcpserver.h index 4fb693c19..a1b02625c 100644 --- a/src/mcpserver.h +++ b/src/mcpserver.h @@ -115,9 +115,17 @@ inline bool sendAll( int fd, const std::string& data ) noexcept std::size_t sent = 0; while( sent < data.size() ) { - const ssize_t n = ::send( fd, data.data() + sent, data.size() - sent, 0 ); + const int toSend = static_cast( std::min( data.size() - sent, 32768 ) ); + const ssize_t n = ::send( static_cast( fd ), data.data() + sent, toSend, 0 ); if( n <= 0 ) { + std::fprintf( stderr, "ripwire-mcp: send failed n=%zd err=%d\n", n, +#ifdef _WIN32 + WSAGetLastError() +#else + errno +#endif + ); return false; } sent += static_cast( n ); diff --git a/src/quality.h b/src/quality.h index 1edae8ac2..c8d12da10 100644 --- a/src/quality.h +++ b/src/quality.h @@ -967,6 +967,37 @@ inline ContentIdIndex contentIdsBySym( const IngestResult& ing, const Graph& g, // unrelated agent-session files. Returns the dir with NO trailing slash. Deterministic per (user, env). inline std::string cacheDirLadder() { +#if defined(_WIN32) + std::string d; + const char* localAppData = std::getenv( "LOCALAPPDATA" ); + const char* tempDir = std::getenv( "TEMP" ); + if( !tempDir ) tempDir = std::getenv( "TMP" ); + + if( localAppData && *localAppData ) + { + d = localAppData; + } + else if( tempDir && *tempDir ) + { + d = tempDir; + } + else + { + d = "C:/Windows/Temp"; + } + while( d.size() > 1 && ( d.back() == '/' || d.back() == '\\' ) ) + { + d.pop_back(); + } + d += "/ripwire"; + ::mkdir( d.c_str(), 0700 ); + struct stat st {}; + if( ::stat( d.c_str(), &st ) == 0 && S_ISDIR( st.st_mode ) ) + { + return d; + } + return "NUL"; +#else std::string d; const char* tmpDir = std::getenv( "TMPDIR" ); if( tmpDir && *tmpDir ) @@ -999,6 +1030,7 @@ inline std::string cacheDirLadder() } } return "/dev/null/ripwire-cache-unavailable"; // unsafe/unusable candidate: make cache I/O fail closed +#endif } // popen a shell command and return its trimmed stdout ("" on any failure — never crashes). THE one copy of diff --git a/src/renamemine.h b/src/renamemine.h index 6a6d37d8a..b945bf69d 100644 --- a/src/renamemine.h +++ b/src/renamemine.h @@ -296,7 +296,11 @@ inline RenameHarvest mineRenamePairs( const std::string& root ) const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " log --no-merges --no-color --no-ext-diff --no-textconv --no-renames" +#ifdef _WIN32 + " --format=\"%x01%H\" -p -U0 2>/dev/null"; +#else " --format='%x01%H' -p -U0 2>/dev/null"; +#endif HashMap votes; detail::HunkBuffer hunk; diff --git a/src/verbs_change.h b/src/verbs_change.h index adb1a9429..41348c110 100644 --- a/src/verbs_change.h +++ b/src/verbs_change.h @@ -778,6 +778,136 @@ std::string runCaptureText( RunCapture& cap ) // group at the cap, and decode the exit honestly. Zero new dependencies — POSIX only (G3/G5). RunCapture runCommandCapture( const std::string& cmd, std::uint32_t timeoutSec ) { +#if defined(_WIN32) + RunCapture cap; + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof( sa ); + sa.bInheritHandle = TRUE; + + HANDLE hRead = NULL, hWrite = NULL; + if( !CreatePipe( &hRead, &hWrite, &sa, 0 ) ) + { + cap.isSpawnFailed = true; + return cap; + } + SetHandleInformation( hRead, HANDLE_FLAG_INHERIT, 0 ); + + HANDLE hJob = CreateJobObjectA( NULL, NULL ); + if( hJob != NULL ) + { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli{}; + jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject( hJob, JobObjectExtendedLimitInformation, &jeli, sizeof( jeli ) ); + } + + STARTUPINFOA si{}; + si.cb = sizeof( si ); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = NULL; + si.hStdOutput = hWrite; + si.hStdError = hWrite; + + PROCESS_INFORMATION pi{}; + std::string fullCmd = "cmd.exe /d /c " + cmd; + std::vector cmdBuf( fullCmd.begin(), fullCmd.end() ); + cmdBuf.push_back( '\0' ); + + const auto t0 = std::chrono::steady_clock::now(); + const auto elapsedMs = [ & ]() -> std::int64_t + { return std::chrono::duration_cast( std::chrono::steady_clock::now() - t0 ).count(); }; + + BOOL ok = CreateProcessA( + NULL, + cmdBuf.data(), + NULL, + NULL, + TRUE, + CREATE_SUSPENDED | CREATE_NO_WINDOW, + NULL, + NULL, + &si, + &pi + ); + + CloseHandle( hWrite ); + + if( !ok ) + { + CloseHandle( hRead ); + if( hJob ) CloseHandle( hJob ); + cap.isSpawnFailed = true; + return cap; + } + + if( hJob ) + { + AssignProcessToJobObject( hJob, pi.hProcess ); + } + ResumeThread( pi.hThread ); + CloseHandle( pi.hThread ); + + const std::int64_t timeoutMs = std::int64_t( timeoutSec ) * 1000; + char buf[ 65536 ]; + + for( ;; ) + { + const std::int64_t nowMs = elapsedMs(); + if( !cap.isTimedOut && nowMs >= timeoutMs ) + { + cap.isTimedOut = true; + if( hJob ) TerminateJobObject( hJob, 1 ); + TerminateProcess( pi.hProcess, 1 ); + } + + DWORD bytesAvail = 0; + if( PeekNamedPipe( hRead, NULL, 0, NULL, &bytesAvail, NULL ) && bytesAvail > 0 ) + { + DWORD bytesRead = 0; + if( ReadFile( hRead, buf, sizeof( buf ), &bytesRead, NULL ) && bytesRead > 0 ) + { + runCaptureAppend( cap, buf, static_cast( bytesRead ) ); + continue; + } + } + + DWORD waitRes = WaitForSingleObject( pi.hProcess, 15 ); + if( waitRes == WAIT_OBJECT_0 || cap.isTimedOut ) + { + // Drain remaining + DWORD bytesRead = 0; + while( PeekNamedPipe( hRead, NULL, 0, NULL, &bytesAvail, NULL ) && bytesAvail > 0 ) + { + if( ReadFile( hRead, buf, sizeof( buf ), &bytesRead, NULL ) && bytesRead > 0 ) + { + runCaptureAppend( cap, buf, static_cast( bytesRead ) ); + } + else + { + break; + } + } + break; + } + } + + DWORD exitCode = 0; + GetExitCodeProcess( pi.hProcess, &exitCode ); + cap.durationMs = static_cast( elapsedMs() ); + if( !cap.isTimedOut ) + { + cap.isExitedNormally = true; + cap.exitCode = static_cast( exitCode ); + } + else + { + cap.termSignal = 9; + } + + CloseHandle( hRead ); + CloseHandle( pi.hProcess ); + if( hJob ) CloseHandle( hJob ); + return cap; +#else RunCapture cap; int fds[2]; if( pipe( fds ) != 0 ) @@ -880,6 +1010,7 @@ RunCapture runCommandCapture( const std::string& cmd, std::uint32_t timeoutSec ) cap.termSignal = WTERMSIG( status ); } return cap; +#endif } // split the captured text into its NON-EMPTY lines (views into `text`) — wsdetail::segmentsOf is the shared diff --git a/src/verbs_doctor.h b/src/verbs_doctor.h index 5c4141348..a91996be0 100644 --- a/src/verbs_doctor.h +++ b/src/verbs_doctor.h @@ -573,7 +573,11 @@ int runDoctor( const rw::Config& cfg, const char* argv0 ) // `which ripwire`'s) ---- { const std::string selfPath = selfExecutablePath( argv0 ); +#if defined(_WIN32) + const std::string whichPath = doctorPopenTrim( "where ripwire.exe 2>NUL" ); +#else const std::string whichPath = doctorPopenTrim( "which ripwire 2>/dev/null" ); +#endif struct stat selfSt {}; struct stat whichSt {}; const bool haveSelf = !selfPath.empty() && ::stat( selfPath.c_str(), &selfSt ) == 0; From f4e53e2b72828738325e8361aaea6ff9b411fb38 Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:28:51 -0300 Subject: [PATCH 02/13] fix: address CodeRabbit review findings on Windows port - ci.yml: declare workflow read permissions, disable credential persistence on checkouts, and build both ripwire and ripwire_probe in Windows job - PortableFlags.cmake: prioritize RIPWIRE_NATIVE, support /clang:-march=native for clang-cl, and use /fp:precise to keep isnan/isfinite checks live - verbs_change.h: pass resolved system cmd.exe as lpApplicationName in CreateProcessA - platform_compat.cpp: propagate fflush errors before publishing buffers in rw_fflush and rw_fclose; simplify rw_close for CRT fds - platform_compat.h/mcpserver.h: use pointer-width socket_t and rw_closesocket to preserve 64-bit SOCKET handles across MCP HTTP listener - platform_compat.h: guard std::format_string alias with __cpp_lib_format < 202207L - ingest_cache.h: declare non-const CacheFrame prev in saveCache and drop const_cast - quality.h: create Windows cache directory with restricted DACL and validate process owner SID - jsonesc.h / crossref.h / gitoracle.h / renamemine.h: escape % and git format placeholders to prevent cmd.exe env expansion - portablebuildcheck.sh: handle Git Bash Windows paths seamlessly --- .github/workflows/ci.yml | 10 +++- cmake/PortableFlags.cmake | 18 ++++++-- src/crossref.h | 2 +- src/gitoracle.h | 2 +- src/infra/jsonesc.h | 12 ++++- src/infra/platform_compat.cpp | 26 +++++------ src/infra/platform_compat.h | 23 +++++++-- src/ingest_cache.h | 4 +- src/mcpserver.h | 26 +++++------ src/quality.h | 87 +++++++++++++++++++++++++++++++++-- src/renamemine.h | 2 +- src/verbs_change.h | 10 +++- test/portablebuildcheck.sh | 3 +- 13 files changed, 176 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccd4176e2..6850cb870 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,9 @@ concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + # Never RIPWIRE_NATIVE in CI: -march=native would bake in whatever ISA the CI runner's host happens to # expose that week, defeating the entire point of testing the portable default. @@ -78,6 +81,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # churn/co-change gates read real git history — a shallow clone reddens them + persist-credentials: false - name: Install clang-format / clang-tidy (PINNED major — see the job comment) run: | @@ -151,6 +155,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # churn/co-change gates read real git history — a shallow clone reddens them + persist-credentials: false # clang is installed on BOTH Linux legs, not only the clang one. optremarkscheck's Clang-only # configure arms pin `clang++` when the default front end is not Clang, so the gcc leg keeps that @@ -231,6 +236,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false # RHEL 9's default gcc is 11, which does not implement C++23 — CMakeLists sets CXX_STANDARD 23 with # STANDARD_REQUIRED ON, so the configure would fail outright. gcc-toolset-N is Red Hat's own supported @@ -297,12 +303,13 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Configure (portable — clang + ninja) run: cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ - name: Build - run: cmake --build build -j --target ripwire + run: cmake --build build -j - name: Doctor check run: .\build\ripwire.exe . --doctor @@ -348,6 +355,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # churn/co-change gates read real git history — a shallow clone reddens them + persist-credentials: false - name: Install tooling (Linux) if: runner.os == 'Linux' diff --git a/cmake/PortableFlags.cmake b/cmake/PortableFlags.cmake index 1bfe8298e..1e92314a3 100644 --- a/cmake/PortableFlags.cmake +++ b/cmake/PortableFlags.cmake @@ -34,11 +34,19 @@ if(APPLE AND NOT RIPWIRE_PRETEND_LINUX AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm set(RIPWIRE_IS_APPLE_SILICON ON) endif() -if(MSVC) - # MSVC compiler flags: fast math, conformant C++ mode, UTF-8 source/exec charset - set(RIPWIRE_ARCH_FLAGS /O2 /fp:fast /permissive- /utf-8) -elseif(RIPWIRE_NATIVE) - set(RIPWIRE_ARCH_FLAGS -O3 -march=native -ffast-math -fno-finite-math-only) +if(RIPWIRE_NATIVE) + if(MSVC) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(RIPWIRE_ARCH_FLAGS /O3 /clang:-march=native /fp:precise /permissive- /utf-8) + else() + set(RIPWIRE_ARCH_FLAGS /O2 /fp:precise /permissive- /utf-8) + endif() + else() + set(RIPWIRE_ARCH_FLAGS -O3 -march=native -ffast-math -fno-finite-math-only) + endif() +elseif(MSVC) + # MSVC compiler flags: precise math (preserves isnan/isfinite), conformant C++ mode, UTF-8 source/exec charset + set(RIPWIRE_ARCH_FLAGS /O2 /fp:precise /permissive- /utf-8) elseif(RIPWIRE_IS_APPLE_SILICON) set(RIPWIRE_ARCH_FLAGS -O2 -mcpu=apple-m1 -ffast-math -fno-finite-math-only) else() diff --git a/src/crossref.h b/src/crossref.h index 50bba1591..3b1e4dc73 100644 --- a/src/crossref.h +++ b/src/crossref.h @@ -696,7 +696,7 @@ inline std::vector enumerateRefs( const std::string& root, std::string_ std::size_t* filterNameHits = nullptr ) { #ifdef _WIN32 - const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format=\"%(refname:short)|%(objectname)|%(committerdate:short)\" refs/heads 2>/dev/null" ); + const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format=^%(refname:short^)^|^%(objectname^)^|^%(committerdate:short^) refs/heads 2>/dev/null" ); #else const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format='%(refname:short)|%(objectname)|%(committerdate:short)' refs/heads 2>/dev/null" ); #endif diff --git a/src/gitoracle.h b/src/gitoracle.h index 5e79f6820..4f38dbdc8 100644 --- a/src/gitoracle.h +++ b/src/gitoracle.h @@ -596,7 +596,7 @@ inline HistoryIndex runProbe( const std::string& root ) const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " log --no-merges --no-color --no-ext-diff --no-textconv --no-renames" #ifdef _WIN32 - " --format=\"%x01%H %cs\" -p -U0 2>/dev/null"; + " ^\"--format=^%x01^%H %cs^\" -p -U0 2>/dev/null"; #else " --format='%x01%H %cs' -p -U0 2>/dev/null"; #endif diff --git a/src/infra/jsonesc.h b/src/infra/jsonesc.h index 1427f9b49..f3efb4c4c 100644 --- a/src/infra/jsonesc.h +++ b/src/infra/jsonesc.h @@ -268,10 +268,20 @@ inline bool isJsonWs( char c ) noexcept inline std::string shSingleQuote( const std::string& s ) { #if defined(_WIN32) + // On Windows, cmd.exe /c expands %VAR% inside double quotes (e.g. C:\src\100%repo%). + // Closing the quote, escaping % as ^%, and reopening the quote ("foo"^%"bar") prevents cmd.exe + // from expanding the variable, while CommandLineToArgvW joins the segments into foo%bar. std::string out = "\""; for( char c : s ) { - if( c == '"' ) { out += "\\\""; } + if( c == '"' ) + { + out += "\\\""; + } + else if( c == '%' ) + { + out += "\"^%\""; + } else { out += c; diff --git a/src/infra/platform_compat.cpp b/src/infra/platform_compat.cpp index 9c6eca5b5..b61f3873b 100644 --- a/src/infra/platform_compat.cpp +++ b/src/infra/platform_compat.cpp @@ -336,7 +336,11 @@ int rw_fflush( std::FILE* stream ) if( it != s_memstreams.end() ) { auto& info = it->second; - ::fflush( stream ); + const int flushRes = ::fflush( stream ); + if( flushRes != 0 ) + { + return flushRes; + } long currentPos = std::ftell( stream ); std::fseek( stream, 0, SEEK_END ); long len = std::ftell( stream ); @@ -386,7 +390,12 @@ int rw_fclose( std::FILE* stream ) if( isMem ) { - ::fflush( stream ); + const int flushRes = ::fflush( stream ); + if( flushRes != 0 ) + { + ::fclose( stream ); + return flushRes; + } std::fseek( stream, 0, SEEK_END ); long len = std::ftell( stream ); if( len < 0 ) @@ -419,18 +428,7 @@ int rw_close( int fd ) { return -1; } - if( closesocket( static_cast( fd ) ) == 0 ) - { - return 0; - } - if( WSAGetLastError() == WSAENOTSOCK ) - { - if( _get_osfhandle( fd ) != -1 ) - { - return _close( fd ); - } - } - return -1; + return _close( fd ); } struct WinsockAutoInit diff --git a/src/infra/platform_compat.h b/src/infra/platform_compat.h index 9ec0f4635..7a9fa3912 100644 --- a/src/infra/platform_compat.h +++ b/src/infra/platform_compat.h @@ -203,17 +203,25 @@ return _commit( fd ); } + using socket_t = SOCKET; + #define RW_INVALID_SOCKET INVALID_SOCKET + + inline int rw_closesocket( SOCKET s ) noexcept + { + return ::closesocket( s ); + } + namespace rw::compat { - inline int rw_setsockopt( int s, int level, int optname, const void* optval, int optlen ) + inline int rw_setsockopt( SOCKET s, int level, int optname, const void* optval, int optlen ) { if( level == SOL_SOCKET && optname == SO_RCVTIMEO && optlen == sizeof( timeval ) ) { const auto* tv = static_cast( optval ); DWORD ms = static_cast( tv->tv_sec * 1000 + tv->tv_usec / 1000 ); - return ::setsockopt( static_cast( s ), level, optname, reinterpret_cast( &ms ), sizeof( ms ) ); + return ::setsockopt( s, level, optname, reinterpret_cast( &ms ), sizeof( ms ) ); } - return ::setsockopt( static_cast( s ), level, optname, static_cast( optval ), optlen ); + return ::setsockopt( s, level, optname, static_cast( optval ), optlen ); } } @@ -226,7 +234,7 @@ #include namespace std { - #if defined(_MSC_VER) && !defined(__cpp_lib_format_ranges) + #if defined(_MSC_VER) && ( !defined(__cpp_lib_format) || __cpp_lib_format < 202207L ) template using format_string = _Fmt_string<_Args...>; #endif @@ -278,6 +286,13 @@ return {}; } } + + using socket_t = int; + #define RW_INVALID_SOCKET ( -1 ) + inline int rw_closesocket( int s ) noexcept + { + return ::close( s ); + } #endif #endif diff --git a/src/ingest_cache.h b/src/ingest_cache.h index 2f2e7598e..7968f55b8 100644 --- a/src/ingest_cache.h +++ b/src/ingest_cache.h @@ -1968,7 +1968,7 @@ inline void saveCache( const std::string& path, std::string_view rootDir, const // validate — absent, foreign version/parserVer/arch, torn — CARRY is simply empty and this run // writes its own file set, which is exactly v14's behaviour and self-heals on the next wider run. const CachePathKeys keys = buildCachePathKeys( files, rootDir ); - const CacheFrame prev = openCacheFrame( path, captureValueUses ); + CacheFrame prev = openCacheFrame( path, captureValueUses ); std::vector carry; const std::vector plan = buildCacheWritePlan( keys.order, keys.pathHashes, prev.entries, carry ); @@ -2140,7 +2140,7 @@ inline void saveCache( const std::string& path, std::string_view rootDir, const PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: offset table + trailer" ); finishCacheBlob( w, table ); } - const_cast( prev ).close(); + prev.close(); PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: write + rename" ); // unique per-process temp so two concurrent runs (this repo runs ~20 parallel sessions) don't diff --git a/src/mcpserver.h b/src/mcpserver.h index a1b02625c..8fa6a2188 100644 --- a/src/mcpserver.h +++ b/src/mcpserver.h @@ -110,13 +110,13 @@ inline std::string_view trim( std::string_view s ) noexcept } // send an entire buffer, tolerating short writes; false if the peer went away mid-write (we just drop it). -inline bool sendAll( int fd, const std::string& data ) noexcept +inline bool sendAll( socket_t fd, const std::string& data ) noexcept { std::size_t sent = 0; while( sent < data.size() ) { const int toSend = static_cast( std::min( data.size() - sent, 32768 ) ); - const ssize_t n = ::send( static_cast( fd ), data.data() + sent, toSend, 0 ); + const ssize_t n = ::send( fd, data.data() + sent, toSend, 0 ); if( n <= 0 ) { std::fprintf( stderr, "ripwire-mcp: send failed n=%zd err=%d\n", n, @@ -134,7 +134,7 @@ inline bool sendAll( int fd, const std::string& data ) noexcept } // build + send a minimal HTTP/1.1 response. Connection: close — one request per connection (§2b serialize). -inline void respond( int fd, const char* status, const char* contentType, const std::string& body ) noexcept +inline void respond( socket_t fd, const char* status, const char* contentType, const std::string& body ) noexcept { std::string out; out.reserve( body.size() + 160 ); @@ -184,7 +184,7 @@ struct Request // makes recv() return <= 0 → we abandon the connection (server lives). // // `tooManyHeaderBytes` / `tooLargeBody` out-params let the caller pick the right 4xx without a wider enum. -inline Request readRequest( int fd, bool& tooManyHeaderBytes, bool& tooLargeBody ) +inline Request readRequest( socket_t fd, bool& tooManyHeaderBytes, bool& tooLargeBody ) { tooManyHeaderBytes = false; tooLargeBody = false; @@ -516,8 +516,8 @@ inline int runMcpHttp( const McpHttpConfig& cfg ) && !gitRepoToplevel( pinnedRoot ).empty(); // ── 4) open the listening socket ─────────────────────────────────────────────────────────────────── - const int listenFd = ::socket( AF_INET, SOCK_STREAM, 0 ); - if( listenFd < 0 ) { std::fprintf( stderr, "ripwire: --listen: socket() failed: %s\n", std::strerror( errno ) ); return 1; } + const socket_t listenFd = ::socket( AF_INET, SOCK_STREAM, 0 ); + if( listenFd == RW_INVALID_SOCKET ) { std::fprintf( stderr, "ripwire: --listen: socket() failed: %s\n", std::strerror( errno ) ); return 1; } int one = 1; ::setsockopt( listenFd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof( one ) ); @@ -528,19 +528,19 @@ inline int runMcpHttp( const McpHttpConfig& cfg ) if( ::inet_pton( AF_INET, bindHost.c_str(), &addr.sin_addr ) != 1 ) { std::fprintf( stderr, "ripwire: --listen: '%s' is not a valid IPv4 bind address (IPv6 is not supported; reverse-proxy for that)\n", host.c_str() ); - ::close( listenFd ); + rw_closesocket( listenFd ); return 1; } if( ::bind( listenFd, reinterpret_cast( &addr ), sizeof( addr ) ) != 0 ) { std::fprintf( stderr, "ripwire: --listen: bind %s:%d failed: %s\n", host.c_str(), port, std::strerror( errno ) ); - ::close( listenFd ); + rw_closesocket( listenFd ); return 1; } if( ::listen( listenFd, 16 ) != 0 ) { std::fprintf( stderr, "ripwire: --listen: listen() failed: %s\n", std::strerror( errno ) ); - ::close( listenFd ); + rw_closesocket( listenFd ); return 1; } @@ -577,8 +577,8 @@ inline int runMcpHttp( const McpHttpConfig& cfg ) // ── 6) accept loop: single-threaded, one request per connection (Connection: close) — §2b serialize ─ for( ;; ) { - const int fd = ::accept( listenFd, nullptr, nullptr ); - if( fd < 0 ) + const socket_t fd = ::accept( listenFd, nullptr, nullptr ); + if( fd == RW_INVALID_SOCKET ) { if( errno == EINTR ) { @@ -678,11 +678,11 @@ inline int runMcpHttp( const McpHttpConfig& cfg ) } } - ::close( fd ); + rw_closesocket( fd ); } // unreachable (the accept loop runs until the process is signalled) — kept for symmetry / future signal handling. - ::close( listenFd ); + rw_closesocket( listenFd ); return 0; } diff --git a/src/quality.h b/src/quality.h index c8d12da10..ffdf8802a 100644 --- a/src/quality.h +++ b/src/quality.h @@ -38,6 +38,11 @@ #include // EWOULDBLOCK — the LOCK_NB retry predicate #include // ::nanosleep — the lock's bounded 10 ms poll +#if defined(_WIN32) +#include +#include +#endif + #include #include // Phase-M: the tmp-name sequence counter (atomicWriteFile); also the A5 process-once cache-sweep guard #include // std::isxdigit/std::isdigit — B10.2d churn-blame porcelain parsing @@ -990,9 +995,85 @@ inline std::string cacheDirLadder() d.pop_back(); } d += "/ripwire"; - ::mkdir( d.c_str(), 0700 ); - struct stat st {}; - if( ::stat( d.c_str(), &st ) == 0 && S_ISDIR( st.st_mode ) ) + + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof( sa ); + sa.bInheritHandle = FALSE; + PSECURITY_DESCRIPTOR pSD = nullptr; + if( ConvertStringSecurityDescriptorToSecurityDescriptorA( + "D:P(A;OICI;GA;;;OW)(A;OICI;GA;;;BA)", + SDDL_REVISION_1, + &pSD, + nullptr ) ) + { + sa.lpSecurityDescriptor = pSD; + } + + CreateDirectoryA( d.c_str(), pSD ? &sa : nullptr ); + if( pSD ) + { + LocalFree( pSD ); + } + + const DWORD attrs = GetFileAttributesA( d.c_str() ); + if( attrs == INVALID_FILE_ATTRIBUTES || !( attrs & FILE_ATTRIBUTE_DIRECTORY ) ) + { + return "NUL"; + } + + HANDLE hToken = NULL; + if( !OpenProcessToken( GetCurrentProcess(), TOKEN_QUERY, &hToken ) ) + { + return "NUL"; + } + + BYTE tokenBuf[ 256 ]; + DWORD tokenLen = 0; + GetTokenInformation( hToken, TokenUser, tokenBuf, sizeof( tokenBuf ), &tokenLen ); + const TOKEN_USER* pTokenUser = reinterpret_cast( tokenBuf ); + const PSID userSid = pTokenUser ? pTokenUser->User.Sid : nullptr; + + PSID pSidOwner = nullptr; + PSECURITY_DESCRIPTOR pSDGet = nullptr; + const DWORD res = GetNamedSecurityInfoA( + d.c_str(), + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION, + &pSidOwner, + nullptr, + nullptr, + nullptr, + &pSDGet ); + + bool ownerMatch = false; + if( res == ERROR_SUCCESS && pSidOwner && userSid ) + { + if( EqualSid( pSidOwner, userSid ) ) + { + ownerMatch = true; + } + else + { + SID_IDENTIFIER_AUTHORITY ntAuth = SECURITY_NT_AUTHORITY; + PSID adminSid = nullptr; + if( AllocateAndInitializeSid( &ntAuth, 2, SECURITY_BUILTIN_DOMAIN_RID, DOMAIN_ALIAS_RID_ADMINS, 0, 0, 0, 0, 0, 0, &adminSid ) ) + { + if( EqualSid( pSidOwner, adminSid ) ) + { + ownerMatch = true; + } + FreeSid( adminSid ); + } + } + } + + if( pSDGet ) + { + LocalFree( pSDGet ); + } + CloseHandle( hToken ); + + if( ownerMatch ) { return d; } diff --git a/src/renamemine.h b/src/renamemine.h index b945bf69d..aa98a2f0b 100644 --- a/src/renamemine.h +++ b/src/renamemine.h @@ -297,7 +297,7 @@ inline RenameHarvest mineRenamePairs( const std::string& root ) const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " log --no-merges --no-color --no-ext-diff --no-textconv --no-renames" #ifdef _WIN32 - " --format=\"%x01%H\" -p -U0 2>/dev/null"; + " ^\"--format=^%x01^%H^\" -p -U0 2>/dev/null"; #else " --format='%x01%H' -p -U0 2>/dev/null"; #endif diff --git a/src/verbs_change.h b/src/verbs_change.h index 41348c110..c7c4ec3df 100644 --- a/src/verbs_change.h +++ b/src/verbs_change.h @@ -807,8 +807,14 @@ RunCapture runCommandCapture( const std::string& cmd, std::uint32_t timeoutSec ) si.hStdOutput = hWrite; si.hStdError = hWrite; + char sysDir[ MAX_PATH ]; + const UINT sysDirLen = GetSystemDirectoryA( sysDir, MAX_PATH ); + const std::string cmdExePath = ( sysDirLen > 0 && sysDirLen < MAX_PATH ) + ? std::string( sysDir ) + "\\cmd.exe" + : "C:\\Windows\\System32\\cmd.exe"; + PROCESS_INFORMATION pi{}; - std::string fullCmd = "cmd.exe /d /c " + cmd; + std::string fullCmd = "\"" + cmdExePath + "\" /d /c " + cmd; std::vector cmdBuf( fullCmd.begin(), fullCmd.end() ); cmdBuf.push_back( '\0' ); @@ -817,7 +823,7 @@ RunCapture runCommandCapture( const std::string& cmd, std::uint32_t timeoutSec ) { return std::chrono::duration_cast( std::chrono::steady_clock::now() - t0 ).count(); }; BOOL ok = CreateProcessA( - NULL, + cmdExePath.c_str(), cmdBuf.data(), NULL, NULL, diff --git a/test/portablebuildcheck.sh b/test/portablebuildcheck.sh index 8cbaf6bc3..64b00cea8 100755 --- a/test/portablebuildcheck.sh +++ b/test/portablebuildcheck.sh @@ -25,10 +25,11 @@ # Exits non-zero on any failure; prints PASS/FAIL per check, ALL PASS on success. set -u -ROOT="$( cd "$( dirname "$0" )/.." && pwd )" +ROOT="$( cd "$( dirname "$0" )/.." && ( pwd -W 2>/dev/null || pwd ) )" MODULE="$ROOT/cmake/PortableFlags.cmake" CMAKE_TOP="$ROOT/CMakeLists.txt" TMP="$( mktemp -d )" +TMP="$( cd "$TMP" && ( pwd -W 2>/dev/null || pwd ) )" trap 'rm -rf "$TMP"' EXIT fail=0 From 3faf5c22a05a94c813f292559e920d59356b8286 Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:38:04 -0300 Subject: [PATCH 03/13] fix(quality): fail closed if Windows security descriptor creation fails --- src/quality.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/quality.h b/src/quality.h index ffdf8802a..673531ea9 100644 --- a/src/quality.h +++ b/src/quality.h @@ -1009,12 +1009,14 @@ inline std::string cacheDirLadder() sa.lpSecurityDescriptor = pSD; } - CreateDirectoryA( d.c_str(), pSD ? &sa : nullptr ); - if( pSD ) + if( !pSD ) { - LocalFree( pSD ); + return "NUL"; } + CreateDirectoryA( d.c_str(), &sa ); + LocalFree( pSD ); + const DWORD attrs = GetFileAttributesA( d.c_str() ); if( attrs == INVALID_FILE_ATTRIBUTES || !( attrs & FILE_ATTRIBUTE_DIRECTORY ) ) { From 5ae71a436a2a756fd1d66280525582eb1d84344c Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:59:02 -0300 Subject: [PATCH 04/13] build(windows): support Release LTO and PGO builds on Windows - Adapt scripts/pgobuild.sh for Windows (.exe binary suffix and llvm-profdata.exe candidate lookup) - Document Release mode with ThinLTO and PGO build procedures in CONTRIBUTING.md --- CONTRIBUTING.md | 9 +++++++++ scripts/pgobuild.sh | 17 ++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a2498b4c6..c1530c93f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,15 @@ cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang cmake --build build -j ``` +For maximum performance (Release mode with ThinLTO and host-CPU vectorization): + +```cmd +cmake -S . -B build-release -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DRIPWIRE_NATIVE=ON +cmake --build build-release -j +``` + +Profile-Guided Optimization (PGO) is also supported on Windows via `scripts/pgobuild.sh` (under Git Bash) or CMake (`-DRIPWIRE_PGO=generate` and `-DRIPWIRE_PGO=use`), providing an additional 2–11% speedup across hot capture, query, and AST linting paths. + Or using the Visual Studio generator with Clang-CL: ```cmd diff --git a/scripts/pgobuild.sh b/scripts/pgobuild.sh index bb6c72e3d..d0c741cab 100755 --- a/scripts/pgobuild.sh +++ b/scripts/pgobuild.sh @@ -51,11 +51,11 @@ done command -v cmake >/dev/null || { echo "cmake required" >&2; exit 2; } [ -d "$CORPUS" ] || { echo "training corpus not found: $CORPUS" >&2; exit 2; } -# llvm-profdata: inside the active toolchain on macOS, on PATH (often versioned) on Linux. -PROFDATA="$( xcrun --find llvm-profdata 2>/dev/null || command -v llvm-profdata 2>/dev/null || true )" +# llvm-profdata: inside the active toolchain on macOS, on PATH (often versioned) on Linux, or with .exe on Windows. +PROFDATA="$( xcrun --find llvm-profdata 2>/dev/null || command -v llvm-profdata 2>/dev/null || command -v llvm-profdata.exe 2>/dev/null || true )" if [ -z "$PROFDATA" ]; then - for v in 21 20 19 18 17; do - cand="$( command -v "llvm-profdata-$v" 2>/dev/null || true )" + for v in 22 21 20 19 18 17; do + cand="$( command -v "llvm-profdata-$v" 2>/dev/null || command -v "llvm-profdata-$v.exe" 2>/dev/null || true )" [ -n "$cand" ] && { PROFDATA="$cand"; break; } done fi @@ -75,6 +75,7 @@ if [ "$reuse" -eq 0 ]; then # ── phase 2: train ───────────────────────────────────────────────────────────────────────────── echo "pgobuild: [3/4] training on $CORPUS" BIN="$GEN/ripwire" + [ -f "$BIN.exe" ] && BIN="$BIN.exe" TRAIN="$( mktemp -d )" run(){ "$BIN" "$@" >/dev/null 2>&1 || echo "pgobuild: training run returned non-zero (continuing): $*" >&2; } run "$CORPUS" --no-cache @@ -112,7 +113,9 @@ cmake --build "$OPT" -j "$JOBS" >"$OPT/build.log" 2>&1 || { echo "optimized build failed — see $OPT/build.log" >&2; tail -20 "$OPT/build.log" >&2; exit 1; } echo "pgobuild: done — $OPT/ripwire (profile: $PROFILE)" +OPT_BIN="$OPT/ripwire" +[ -f "$OPT_BIN.exe" ] && OPT_BIN="$OPT_BIN.exe" echo "pgobuild: verify before you trust it:" -echo " $OPT/ripwire $ROOT >a; $OPT/ripwire $ROOT >b; diff -q a b # determinism is a contract" -echo " diff -q <($OPT/ripwire $ROOT) <($ROOT/build/ripwire $ROOT) # PGO must not change a byte of output" -echo " RIPWIRE_BIN=$OPT/ripwire python3 $ROOT/test/pargates.py $ROOT $OPT/ripwire -j 6" +echo " $OPT_BIN $ROOT >a; $OPT_BIN $ROOT >b; diff -q a b # determinism is a contract" +echo " diff -q <($OPT_BIN $ROOT) <($ROOT/build/ripwire $ROOT) # PGO must not change a byte of output" +echo " RIPWIRE_BIN=$OPT_BIN python3 $ROOT/test/pargates.py $ROOT $OPT_BIN -j 6" From 29bf9b91c072bb0da8b43002b67c2dcf7f3af3d8 Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:09:14 -0300 Subject: [PATCH 05/13] build(pgo): resolve baseline binary path with .exe fallback in verification output Apply .exe fallback to BASE_BIN so the emitted PGO diff verification command finds the Windows baseline executable correctly. --- scripts/pgobuild.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/pgobuild.sh b/scripts/pgobuild.sh index d0c741cab..24de5c466 100755 --- a/scripts/pgobuild.sh +++ b/scripts/pgobuild.sh @@ -115,7 +115,9 @@ cmake --build "$OPT" -j "$JOBS" >"$OPT/build.log" 2>&1 || { echo "pgobuild: done — $OPT/ripwire (profile: $PROFILE)" OPT_BIN="$OPT/ripwire" [ -f "$OPT_BIN.exe" ] && OPT_BIN="$OPT_BIN.exe" +BASE_BIN="$ROOT/build/ripwire" +[ -f "$BASE_BIN.exe" ] && BASE_BIN="$BASE_BIN.exe" echo "pgobuild: verify before you trust it:" echo " $OPT_BIN $ROOT >a; $OPT_BIN $ROOT >b; diff -q a b # determinism is a contract" -echo " diff -q <($OPT_BIN $ROOT) <($ROOT/build/ripwire $ROOT) # PGO must not change a byte of output" +echo " diff -q <($OPT_BIN $ROOT) <($BASE_BIN $ROOT) # PGO must not change a byte of output" echo " RIPWIRE_BIN=$OPT_BIN python3 $ROOT/test/pargates.py $ROOT $OPT_BIN -j 6" From d085af992a10d3ee84bda34aceebd3fb7fe07547 Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:52:34 -0300 Subject: [PATCH 06/13] fix(windows): bound concurrent scans and document port --- src/crossref.h | 2 ++ src/gitmine.h | 7 ++-- src/gitoracle.h | 1 + src/infra/jsonesc.h | 1 + src/infra/platform_compat.cpp | 14 ++++++++ src/infra/platform_compat.h | 19 ++++++++++ src/infra/processLock.h | 67 +++++++++++++++++++++++++++++++++++ src/ingest.cpp | 21 +++++++++-- src/ingest.h | 2 +- src/ingest_cache.h | 3 ++ src/ingest_parsepool.h | 5 +++ src/main.cpp | 39 ++++++++++++++++++++ src/mcpserver.h | 5 +++ src/quality.h | 5 +-- src/renamemine.h | 1 + src/verbs_change.h | 1 + src/verbs_doctor.h | 1 + test/skipreasoncheck.sh | 34 +++++++++++------- 18 files changed, 208 insertions(+), 20 deletions(-) create mode 100644 src/infra/processLock.h diff --git a/src/crossref.h b/src/crossref.h index 3b1e4dc73..d6004aa8a 100644 --- a/src/crossref.h +++ b/src/crossref.h @@ -460,6 +460,7 @@ struct StreamBlobStats } }; +/// Streams requested git objects in one framed batch while bounding buffered blob memory. template inline void streamBlobs( const std::string& root, const std::vector& shas, OnBlob onBlob, StreamBlobStats* stats = nullptr ) @@ -692,6 +693,7 @@ struct RefInfo // out.size(): a filter matching only the checked-out branch has SELECTED something (the answer is "nothing // but the ref you are on"), while a filter matching no branch name at all has selected nothing and must // refuse rather than report refs="0" — which reads as "no branch carries stray work". +/// Enumerates local branches in deterministic order and excludes the checked-out ref from stray-content results. inline std::vector enumerateRefs( const std::string& root, std::string_view filter, const std::string& headSha, std::size_t* filterNameHits = nullptr ) { diff --git a/src/gitmine.h b/src/gitmine.h index 35c611fa2..1dfec504d 100644 --- a/src/gitmine.h +++ b/src/gitmine.h @@ -1475,11 +1475,12 @@ inline RawCommitStream gitLogNameOnlyRaw( const std::string& root, const std::st inline std::string gitWindowBoundarySha( const std::string& root, const std::string& coSince ) { PROFILE_SCOPE_DESCRIBE( "gitmine: gitWindowBoundarySha (cheap window-drift probe)" ); - // G3: the shared reader, not a private `char buf[128]` + fgets accumulate. `| tail -1` already reduces - // the output to one line, so `.back()` is that line; gitCommandLines has already stripped its CR/LF tail. + // G3: the shared reader, not a private `char buf[128]` + fgets accumulate. Git's reverse/max-count + // options reduce the output to one line, so `back()` is that line; gitCommandLines has already stripped + // its CR/LF tail. const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " log --since=" + shSingleQuote( defaultWindowSince( root, coSince ) ) // F1: the probe must resolve the window it guards, by the same rule - + " --format=%H 2>/dev/null | tail -1"; + + " --format=%H --reverse --max-count=1 2>/dev/null"; const GitCommandLines res = gitCommandLines( cmd ); if( !res.isStarted || res.lines.empty() ) { diff --git a/src/gitoracle.h b/src/gitoracle.h index 4f38dbdc8..e8b7aebf8 100644 --- a/src/gitoracle.h +++ b/src/gitoracle.h @@ -589,6 +589,7 @@ inline PatchWalk walkGitPatch( const std::string& cmd, OnLine onLine, KeepWalkin // a \x01-led header line per commit. \x01 cannot appear in a unified-diff marker column, so // the framing is unambiguous without a second pass. %cs is git's COMMITTER date — the same // deterministic clock quality::gitCommitterDateIso uses; the wall clock is never consulted. +/// Runs the bounded git history probe and returns an honest, possibly truncated name-removal index. inline HistoryIndex runProbe( const std::string& root ) { HistoryIndex idx; diff --git a/src/infra/jsonesc.h b/src/infra/jsonesc.h index f3efb4c4c..007d5d85c 100644 --- a/src/infra/jsonesc.h +++ b/src/infra/jsonesc.h @@ -265,6 +265,7 @@ inline bool isJsonWs( char c ) noexcept // it in for free. gitmine.h's rw::shSingleQuote is the more widely used name (main.cpp, prcontext.h, // quality.h, mcp server) — kept as the canonical spelling; docparse.h's detail::shellQuote now // forwards here instead of carrying its own copy. +/// Quotes a command argument for the active shell without allowing path data to expand variables. inline std::string shSingleQuote( const std::string& s ) { #if defined(_WIN32) diff --git a/src/infra/platform_compat.cpp b/src/infra/platform_compat.cpp index b61f3873b..4957c6c0a 100644 --- a/src/infra/platform_compat.cpp +++ b/src/infra/platform_compat.cpp @@ -24,6 +24,7 @@ namespace rw::compat { +/// Maps POSIX advisory locking flags to Win32 byte-range locks and reports failures through errno. int rw_flock( int fd, int operation ) noexcept { HANDLE hFile = reinterpret_cast( _get_osfhandle( fd ) ); @@ -73,6 +74,7 @@ int rw_flock( int fd, int operation ) noexcept return 0; } +/// Reads a byte range at an explicit offset without changing the descriptor's shared file position. ssize_t rw_pread( int fd, void* buf, std::size_t count, std::uint64_t offset ) noexcept { HANDLE hFile = reinterpret_cast( _get_osfhandle( fd ) ); @@ -101,6 +103,7 @@ ssize_t rw_pread( int fd, void* buf, std::size_t count, std::uint64_t offset ) n return static_cast( bytesRead ); } +/// Resolves a Windows path through the CRT equivalent of POSIX realpath. char* rw_realpath( const char* path, char* resolved_path ) noexcept { if( path == nullptr ) @@ -111,6 +114,7 @@ char* rw_realpath( const char* path, char* resolved_path ) noexcept return _fullpath( resolved_path, path, PATH_MAX ); } +/// Runs a shell command with binary pipes and translates POSIX null-device redirection to Windows NUL. std::FILE* rw_popen( const char* command, const char* mode ) { if( command == nullptr || mode == nullptr ) @@ -143,6 +147,7 @@ std::FILE* rw_popen( const char* command, const char* mode ) return _popen( cmd.c_str(), winMode.c_str() ); } +/// Returns an 8.3 path suitable for cmd.exe redirection, or the normalized input when conversion fails. std::string rw_short_path( const std::string& path ) { if( path.empty() ) @@ -166,6 +171,7 @@ std::string rw_short_path( const std::string& path ) return winPath; } +/// Closes a command pipe opened by rw_popen and returns the child-process status. int rw_pclose( std::FILE* stream ) { if( stream == nullptr ) @@ -175,6 +181,7 @@ int rw_pclose( std::FILE* stream ) return _pclose( stream ); } +/// Returns the absolute path of the running executable when Windows can provide it. std::string rw_self_exe_path() { char buf[MAX_PATH]; @@ -187,6 +194,7 @@ std::string rw_self_exe_path() } struct pollfd; +/// Polls inherited Windows pipe handles until input, hangup, an invalid descriptor, or the deadline is seen. int rw_poll( struct pollfd* fds, unsigned long nfds, int timeout ) { if( fds == nullptr || nfds == 0 ) @@ -264,6 +272,7 @@ struct MemStreamInfo static std::mutex s_memstream_mutex; static std::unordered_map s_memstreams; +/// Creates a temporary-file-backed stream with the open_memstream ownership contract. std::FILE* rw_open_memstream( char** bufloc, std::size_t* sizeloc ) { if( bufloc == nullptr || sizeloc == nullptr ) @@ -324,6 +333,7 @@ std::FILE* rw_open_memstream( char** bufloc, std::size_t* sizeloc ) return fp; } +/// Flushes a compatibility memory stream and refreshes its caller-owned buffer and byte count. int rw_fflush( std::FILE* stream ) { if( stream == nullptr ) @@ -368,6 +378,7 @@ int rw_fflush( std::FILE* stream ) return ::fflush( stream ); } +/// Flushes and closes a compatibility memory stream, publishing its final buffer before release. int rw_fclose( std::FILE* stream ) { if( stream == nullptr ) @@ -422,6 +433,7 @@ int rw_fclose( std::FILE* stream ) return ::fclose( stream ); } +/// Closes a CRT file descriptor without confusing it with a Winsock SOCKET. int rw_close( int fd ) { if( fd < 0 ) @@ -433,11 +445,13 @@ int rw_close( int fd ) struct WinsockAutoInit { + /// Initializes Winsock for the process before any socket compatibility wrapper is used. WinsockAutoInit() { WSADATA d; WSAStartup( MAKEWORD( 2, 2 ), &d ); } + /// Balances the process-wide Winsock initialization at normal process teardown. ~WinsockAutoInit() { WSACleanup(); diff --git a/src/infra/platform_compat.h b/src/infra/platform_compat.h index 7a9fa3912..bf62dbd5d 100644 --- a/src/infra/platform_compat.h +++ b/src/infra/platform_compat.h @@ -86,6 +86,7 @@ int rw_fflush( std::FILE* stream ); int rw_close( int fd ); std::string rw_short_path( const std::string& path ); + /// Publishes a replacement file with bounded retries for transient Windows sharing violations. inline int rw_rename( const char* oldname, const char* newname ) noexcept { for( int attempt = 0; attempt < 8; ++attempt ) @@ -147,16 +148,19 @@ #ifndef fflush #define fflush rw_fflush #endif + /// Closes a CRT descriptor while preserving the separate SOCKET close path. inline int close( int fd ) { return rw::compat::rw_close( fd ); } #include + /// Fills a caller-provided tm with local time using the thread-safe MSVC API. inline struct tm* rw_localtime_r( const time_t* timer, struct tm* buf ) noexcept { return localtime_s( buf, timer ) == 0 ? buf : nullptr; } + /// Fills a caller-provided tm with UTC time using the thread-safe MSVC API. inline struct tm* rw_gmtime_r( const time_t* timer, struct tm* buf ) noexcept { return gmtime_s( buf, timer ) == 0 ? buf : nullptr; @@ -168,21 +172,25 @@ #define gmtime_r rw_gmtime_r #endif + /// Adapts POSIX directory creation to the CRT while intentionally ignoring POSIX mode bits. inline int mkdir( const char* path, int /*mode*/ ) { return _mkdir( path ); } + /// Provides the lstat shape used by the POSIX code through the CRT stat result. inline int lstat( const char* path, struct stat* buf ) { return ::stat( path, buf ); } + /// Supplies the stable non-root identity used by cache-path code on Windows. inline unsigned int getuid() noexcept { return 1000; } + /// Sleeps for the requested POSIX timespec duration and preserves the zero-success convention. inline int nanosleep( const struct timespec* req, struct timespec* /*rem*/ ) noexcept { if( req ) @@ -193,11 +201,13 @@ return 0; } + /// Keeps the POSIX permission call harmless where Windows descriptors use a different model. inline int fchmod( int /*fd*/, int /*mode*/ ) noexcept { return 0; } + /// Flushes a Windows CRT descriptor to the underlying file through _commit. inline int fsync( int fd ) noexcept { return _commit( fd ); @@ -206,6 +216,7 @@ using socket_t = SOCKET; #define RW_INVALID_SOCKET INVALID_SOCKET + /// Closes a Winsock handle through closesocket rather than the CRT close function. inline int rw_closesocket( SOCKET s ) noexcept { return ::closesocket( s ); @@ -213,6 +224,7 @@ namespace rw::compat { + /// Converts POSIX timeval receive timeouts to the millisecond form expected by Winsock. inline int rw_setsockopt( SOCKET s, int level, int optname, const void* optval, int optlen ) { if( level == SOL_SOCKET && optname == SO_RCVTIMEO && optlen == sizeof( timeval ) ) @@ -256,31 +268,37 @@ #ifdef __cplusplus namespace rw::compat { + /// Keeps the POSIX build on the same compatibility API by forwarding flock unchanged. inline int rw_flock( int fd, int operation ) noexcept { return ::flock( fd, operation ); } + /// Keeps the POSIX build on the same compatibility API by forwarding pread unchanged. inline ssize_t rw_pread( int fd, void* buf, std::size_t count, std::uint64_t offset ) noexcept { return ::pread( fd, buf, count, static_cast( offset ) ); } + /// Keeps the POSIX build on the same compatibility API by forwarding realpath unchanged. inline char* rw_realpath( const char* path, char* resolved_path ) noexcept { return ::realpath( path, resolved_path ); } + /// Keeps the POSIX build on the same compatibility API by forwarding popen unchanged. inline std::FILE* rw_popen( const char* command, const char* mode ) { return ::popen( command, mode ); } + /// Keeps the POSIX build on the same compatibility API by forwarding pclose unchanged. inline int rw_pclose( std::FILE* stream ) { return ::pclose( stream ); } + /// Returns no executable override on POSIX, where the native path helper is unnecessary. inline std::string rw_self_exe_path() { return {}; @@ -289,6 +307,7 @@ using socket_t = int; #define RW_INVALID_SOCKET ( -1 ) + /// Closes the POSIX socket descriptor through close. inline int rw_closesocket( int s ) noexcept { return ::close( s ); diff --git a/src/infra/processLock.h b/src/infra/processLock.h new file mode 100644 index 000000000..2de805542 --- /dev/null +++ b/src/infra/processLock.h @@ -0,0 +1,67 @@ +#pragma once + +#include "Diagnostics.h" +#include "platform_compat.h" + +#include +#include +#include +#include +#include +#include + +namespace rw::infra +{ + +// A stable lockfile whose inode survives cache publishes. The kernel releases the lock when the owning +// process exits, so a crashed ripwire cannot leave a stale lock that blocks future runs. +class ProcessLock +{ +public: + explicit ProcessLock( const std::string& lockPath ) + { + fd_ = ::open( lockPath.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600 ); + if( fd_ < 0 ) + { + DEGRADED_PATH_ALERT( "ripwire: process lockfile open failed; continuing without cross-process serialization" ); + std::fprintf( stderr, "ripwire: process lock unavailable; continuing without cross-process serialization\n" ); + return; + } + + for( ;; ) + { + if( ::flock( fd_, LOCK_EX ) == 0 ) + { + locked_ = true; + return; + } + if( errno != EINTR ) + { + DEGRADED_PATH_ALERT( "ripwire: process lock acquire failed; continuing without cross-process serialization" ); + std::fprintf( stderr, "ripwire: process lock acquire failed; continuing without cross-process serialization\n" ); + return; + } + } + } + + ~ProcessLock() + { + if( fd_ >= 0 ) + { + if( locked_ ) + { + ::flock( fd_, LOCK_UN ); + } + ::close( fd_ ); + } + } + + ProcessLock( const ProcessLock& ) = delete; + ProcessLock& operator=( const ProcessLock& ) = delete; + +private: + int fd_ = -1; + bool locked_ = false; +}; + +} // namespace rw::infra diff --git a/src/ingest.cpp b/src/ingest.cpp index cff94e055..ee5ff08e7 100644 --- a/src/ingest.cpp +++ b/src/ingest.cpp @@ -275,6 +275,10 @@ IngestResult ingest( const char* rootDir, const std::vector& exclud // the deterministic merge, and the dirty-gated saveCache (ingest_parsepool.h). RawFacts raw = runParsePool( result, rootDir, cacheFile, captureValueUses, cache, cacheStats, scan, prewarm ); + // Cache facts are only needed by the parse pool. Release their map and bucket storage before the model tail + // creates symbols/references, so a warm run does not carry the cache and the assembled model at once. + HashMap().swap( cache ); + result.fileHealth = std::move( scan.health ); // §L1: after saveCache, before the (unmeasured) doc pass // ── doc post-pass (P1-B): every collected document file (notebook/html/csv/…) becomes a docText @@ -299,23 +303,36 @@ IngestResult ingest( const char* rootDir, const std::vector& exclud // 4) attribute each reference to its enclosing definition (innermost span containing it) — the // per-file DefSpanIndex + DefSweep cursor every fact family below shares (ingest_model.h). - const DefSpanIndex spanIndex = buildDefSpanIndex( result, raw.defs ); + DefSpanIndex spanIndex = buildDefSpanIndex( result, raw.defs ); // references: order a uint32 index permutation (radix by startByte), then MOVE each RawRef's strings // into its Reference while the shared sweep attributes fromSymbol (ingest_model.h). - const std::vector refOrder = orderReferences( raw.refs, result.files.size() ); + std::vector refOrder = orderReferences( raw.refs, result.files.size() ); emitReferences( result, raw.refs, refOrder, spanIndex ); + std::vector().swap( refOrder ); + std::vector().swap( raw.refs ); dropFieldDefinitionSites( result, fieldDefs ); // member-variable round: a field's defining assignment is not a use of it + std::vector().swap( fieldDefs ); + + // Symbol names/scopes and the definition spans have been transferred to the model/index; the raw definition + // object array is no longer read after this point. + std::vector().swap( raw.defs ); // P2-D Rule 2 bindings, A4-R5 FFI aliases, B6.3 route defs/uses — each in its deterministic total // order, span-attributed families over the same DefSpanIndex (ingest_model.h). emitBindings( result, raw.binds, spanIndex ); + std::vector().swap( raw.binds ); result.includes = std::move( raw.incs ); // physical dependencies (#include / import), for --deps emitBindingAliases( result, raw.ffis ); emitRouteDefs( result, raw.routeDefs ); emitRouteUses( result, raw.routeUses, spanIndex ); + std::vector().swap( raw.routeUses ); + + // No later model pass needs the containment index. Drop it before the macro/shadow passes, which operate on + // the assembled result and can otherwise overlap its storage with a dead span table. + spanIndex = DefSpanIndex{}; // macro-edges round: the corpus-wide role="macro" retag (model.h). AFTER the model is assembled and // AFTER saveCache (which stores the per-file truth, role=Call) — a #define added in one file must diff --git a/src/ingest.h b/src/ingest.h index e79cfc34e..5f1d35ff8 100644 --- a/src/ingest.h +++ b/src/ingest.h @@ -214,7 +214,7 @@ inline bool looksBinary( std::string_view bytes ) noexcept constexpr std::string_view kCrawlSkipDirs[] = { ".git", ".claude", ".hg", ".svn", "node_modules", "vendor", "third_party", ".cache", "build", "dist", "out", "target", ".venv", "venv", "__pycache__", - ".idea", ".vscode", + ".idea", ".vscode", ".worktrees", ".worktrees-clean", "worktrees", // CMake / compiler-id build dirs (generated stubs, not source — break --around=main etc.) "asan", "build_prof", "CMakeFiles", // Generated output captures (docs/captures/ here): a doc that quotes every verb's output out-scores diff --git a/src/ingest_cache.h b/src/ingest_cache.h index 7968f55b8..a6b071cb3 100644 --- a/src/ingest_cache.h +++ b/src/ingest_cache.h @@ -879,6 +879,7 @@ struct ReadFd ReadFd& operator=( const ReadFd& ) = delete; ReadFd( ReadFd&& other ) noexcept : fd( other.fd ) { other.fd = -1; } ~ReadFd() { if( fd >= 0 ) { ::close( fd ); } } + /// Releases the descriptor early so Windows can publish a replacement cache file. void close() noexcept { if( fd >= 0 ) { ::close( fd ); fd = -1; } } // openOnce, not a move-assignment: the only mutation this type needs is "fill an empty guard", and @@ -944,6 +945,7 @@ struct CacheFrame long long mtimeNs = -1;// the blob's own mtime — the warm-run racy-rule reference bool ok = false; CacheReject reason = CacheReject::Absent; // meaningful only while ok == false + /// Closes the held cache frame before an atomic replacement is attempted. void close() noexcept { blob.close(); } }; @@ -1925,6 +1927,7 @@ inline void finishCacheBlob( ByteW& w, const std::vector& table ) // write the cache atomically (path.tmp → rename); groups the merged raw facts back by file. // T5: `rootDir` is the CURRENT invocation's ingest root — every file key is stored root-relative // (relForHash) rather than verbatim, so the cache blob is committable/portable (see kCacheVersion=3). +/// Persists the cache through a validated carry-forward and a platform-safe atomic publication. inline void saveCache( const std::string& path, std::string_view rootDir, const std::vector& files, const std::vector& fileHash, const std::vector& fileSize, const std::vector& fileMtime, diff --git a/src/ingest_parsepool.h b/src/ingest_parsepool.h index 538167937..64f0e160f 100644 --- a/src/ingest_parsepool.h +++ b/src/ingest_parsepool.h @@ -750,6 +750,11 @@ inline RawFacts runParsePool( IngestResult& result, const char* rootDir, std::st raw = mergeThreadFacts( tFacts ); + // The aggregate owns the moved fact payloads now. Release the per-thread vector storage before the + // cache write and before returning to the model-build tail; keeping these empty-but-capacious vectors + // alive needlessly raises the cold-run peak on large trees. + std::vector().swap( tFacts ); + // P1-15: the SAME drift count, carried out of the function instead of only to stderr. The MCP // server discloses it per incremental pass (`_reingest`), which it could not do from an env-gated // print. Read once here, after the pool join that orders every worker's increment. diff --git a/src/main.cpp b/src/main.cpp index e30b80798..07df40225 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -46,6 +46,7 @@ #include "mcpserver.h" // the optional remote MCP transport (--listen), picked below #include "editplan.h" // CLI-first versioned multi-edit transactions #include "wrap.h" +#include "infra/processLock.h" // serialize concurrent heavy CLI runs for the same root // P8 (L7): the test-gate root's ccx_bar= (situ.h kTestGateCcxBarMirror) is quality.h's kCcxBar — one bar, two spellings, // pinned equal in the one TU that sees both (quality.h is also compiled standalone by the bench/probe targets). @@ -200,6 +201,25 @@ std::string defaultCachePath( const std::string& root, bool captureValueUses ) return resolveCacheBlobPath( cacheDirLadder(), tail ); } +std::string canonicalProcessLockPart( std::string_view path ) +{ + namespace fs = std::filesystem; + std::error_code ec; + const fs::path canonical = fs::weakly_canonical( fs::path( path ), ec ); + return ec ? std::string( path ) : canonical.generic_string(); +} + +std::string ingestProcessLockPath( std::string_view identity ) +{ + char name[ 64 ]; + std::snprintf( name, sizeof( name ), "ripwire-ingest-%016llx.lock", + static_cast( rw::fnv1a64( identity ) ) ); + const std::string lockDir = cacheDirLadder() + "/locks"; + std::error_code ec; + std::filesystem::create_directories( std::filesystem::path( lockDir ), ec ); + return lockDir + "/" + name; +} + // computeHeadSnapshot / gitHeadSha / gitRepoHasHistory / cacheDirLadder now live in quality.h (the // baseline home) and are re-exported via the `using` aliases above — one shared copy for CLI + MCP. @@ -3418,6 +3438,25 @@ static int dispatchMain( const rw::Config& cfg, char** argv ) } } + // The lock spans the complete heavy CLI pipeline, including buildGraph and serialization. Locking only + // ingest would still let two processes overlap during the graph's peak, which is exactly the memory hazard + // this guard is meant to prevent. The identity is the canonical root for a single-root run, or the ordered + // canonical root set for a multi-root run; the lockfile lives in the per-user cache, never in the workspace. + std::string ingestLockIdentity; + if( multiRoot ) + { + for( const WorkspaceRoot& r : ws ) + { + ingestLockIdentity += canonicalProcessLockPart( r.arg ); + ingestLockIdentity.push_back( '\0' ); + } + } + else + { + ingestLockIdentity = canonicalProcessLockPart( root ); + } + const rw::infra::ProcessLock ingestProcessLock( ingestProcessLockPath( ingestLockIdentity ) ); + // --index-out=BASE (both-families amendment): the CI generate-and-exit path. // Cold-parse the tree TWICE — once lean, once rich — writing BASE.lean.ripwirecache and // BASE.rich.ripwirecache, then exit 0 WITHOUT emitting a map. Both families ship because the flagship diff --git a/src/mcpserver.h b/src/mcpserver.h index 8fa6a2188..e7e46322d 100644 --- a/src/mcpserver.h +++ b/src/mcpserver.h @@ -95,6 +95,7 @@ inline bool iEquals( std::string_view a, std::string_view b ) noexcept return true; } +/// Trims HTTP header whitespace without allocating or changing the source buffer. inline std::string_view trim( std::string_view s ) noexcept { std::size_t b = 0, e = s.size(); @@ -110,6 +111,7 @@ inline std::string_view trim( std::string_view s ) noexcept } // send an entire buffer, tolerating short writes; false if the peer went away mid-write (we just drop it). +/// Sends one complete HTTP buffer while keeping each platform's socket-width and error conventions. inline bool sendAll( socket_t fd, const std::string& data ) noexcept { std::size_t sent = 0; @@ -134,6 +136,7 @@ inline bool sendAll( socket_t fd, const std::string& data ) noexcept } // build + send a minimal HTTP/1.1 response. Connection: close — one request per connection (§2b serialize). +/// Builds and sends the single-response envelope used by the serialized MCP connection loop. inline void respond( socket_t fd, const char* status, const char* contentType, const std::string& body ) noexcept { std::string out; @@ -184,6 +187,7 @@ struct Request // makes recv() return <= 0 → we abandon the connection (server lives). // // `tooManyHeaderBytes` / `tooLargeBody` out-params let the caller pick the right 4xx without a wider enum. +/// Parses one bounded HTTP request and degrades malformed or stalled input into a caller-visible status. inline Request readRequest( socket_t fd, bool& tooManyHeaderBytes, bool& tooLargeBody ) { tooManyHeaderBytes = false; @@ -420,6 +424,7 @@ inline bool isLoopbackHost( std::string_view host ) noexcept // serve the remote HTTP transport. Returns the process exit code. REFUSES TO START (returns 1 + stderr) // when the security preconditions are not met; otherwise loops forever, one request at a time. +/// Runs the single-threaded MCP HTTP listener with platform-correct socket ownership and cleanup. inline int runMcpHttp( const McpHttpConfig& cfg ) { using namespace mcphttp; diff --git a/src/quality.h b/src/quality.h index 673531ea9..3788b1db2 100644 --- a/src/quality.h +++ b/src/quality.h @@ -970,6 +970,7 @@ inline ContentIdIndex contentIdsBySym( const IngestResult& ing, const Graph& g, // /tmp/ripwire-, always mode 0700. Keeping our artifacts one level below TMPDIR is a performance // boundary as well as a security one: cache hygiene must never enumerate an unbounded shared TMPDIR full of // unrelated agent-session files. Returns the dir with NO trailing slash. Deterministic per (user, env). +/// Selects and validates the per-user cache directory, using a fail-closed path on ownership errors. inline std::string cacheDirLadder() { #if defined(_WIN32) @@ -1124,7 +1125,7 @@ using rw::popenTrimmed; // Run one short git query against `root` and return its whitespace-trimmed output (expected single-line), or // "" on any failure. The shared shape behind gitHeadSha / gitWindowRefSha — `tail` is everything after -// `git -C ` INCLUDING redirects (so a caller can pipe, e.g. "rev-list HEAD 2>/dev/null | tail -1"). +// `git -C ` INCLUDING redirects; callers must use git's own limiting flags so the command is portable. inline std::string gitOneLine( const std::string& root, const std::string& tail ) { return popenTrimmed( "git -c core.quotepath=false -C " + shSingleQuote( root ) + " " + tail ); @@ -1416,7 +1417,7 @@ inline std::string gitWindowRefSha( const std::string& root, std::uint32_t days return preWindow; } - return gitOneLine( root, "rev-list HEAD 2>/dev/null | tail -1" ); // repo younger than the window → its first commit + return gitOneLine( root, "rev-list --max-count=1 --reverse HEAD 2>/dev/null" ); // repo younger than the window → its first commit } // Does `root` sit in a git repo that HAS at least one commit? A WINDOWLESS probe (no --since), so it is true diff --git a/src/renamemine.h b/src/renamemine.h index aa98a2f0b..84e2a2636 100644 --- a/src/renamemine.h +++ b/src/renamemine.h @@ -285,6 +285,7 @@ inline void foldHunk( HunkBuffer& hunk, HashMap& vot // difference that matters here: --no-renames is what turns a file rename into a delete+add of every line, // which is exactly the shape a MOVE has and a name change does not. A moved file therefore contributes hunks // whose lines pair perfectly and differ nowhere, so it votes for nothing; that is the desired behaviour. +/// Mines deterministic rename-pair evidence from git history without inferring similarity-based renames. inline RenameHarvest mineRenamePairs( const std::string& root ) { RenameHarvest harvest; diff --git a/src/verbs_change.h b/src/verbs_change.h index c7c4ec3df..40eb31c2a 100644 --- a/src/verbs_change.h +++ b/src/verbs_change.h @@ -776,6 +776,7 @@ std::string runCaptureText( RunCapture& cap ) // fork/exec `sh -c CMD` in its own process group, drain the pipe under a poll() deadline, SIGKILL the whole // group at the cap, and decode the exit honestly. Zero new dependencies — POSIX only (G3/G5). +/// Captures a bounded subprocess run while killing its complete process tree on timeout. RunCapture runCommandCapture( const std::string& cmd, std::uint32_t timeoutSec ) { #if defined(_WIN32) diff --git a/src/verbs_doctor.h b/src/verbs_doctor.h index a91996be0..e321c07d4 100644 --- a/src/verbs_doctor.h +++ b/src/verbs_doctor.h @@ -547,6 +547,7 @@ inline DoctorAgentRows doctorAgentRows( const rw::Config& cfg, const char* argv0 return out; } +/// Runs the machine-local doctor checks and emits the complete diagnostic result with honest failure states. int runDoctor( const rw::Config& cfg, const char* argv0 ) { using namespace rw; diff --git a/test/skipreasoncheck.sh b/test/skipreasoncheck.sh index 106781c43..168485dbf 100755 --- a/test/skipreasoncheck.sh +++ b/test/skipreasoncheck.sh @@ -38,6 +38,8 @@ # fits. The --skipped legend is under no such budget and is where the definition lives; unindexed_exts= # had no definition anywhere, which is the half that was genuinely undefined. The arm ALSO pins the # negative — the map legend must stay byte-identical — so a future clause cannot land there silently. +# (10) Windows shell probes do not invoke POSIX-only `tail` — a non-git fixture keeps stderr clean on +# every platform. # # Usage: bash test/skipreasoncheck.sh [RIPWIRE_BIN=path/to/binary] # Exits non-zero on any failure. @@ -145,9 +147,12 @@ esac mkdir -p "$TMP/clean" printf 'int cleanOne( void ) { return 1; }\n' > "$TMP/clean/a.cpp" printf 'def clean_two():\n return 2\n' > "$TMP/clean/b.py" -"$BIN" clean --no-cache > "$TMP/clean.xml" 2>/dev/null +"$BIN" clean --no-cache > "$TMP/clean.xml" 2>"$TMP/clean.err" grep -q 'unindexed=' "$TMP/clean.xml" && no '(5) unindexed= emitted on a fully-indexable tree — not additive' \ || ok '(5) no unindexed= attribute when nothing is unindexed (byte-identical default)' +grep -qiE 'not recognized|command not found' "$TMP/clean.err" \ + && no '(10) a platform-specific shell command leaked into stderr' \ + || ok '(10) no POSIX-only shell command leaked into stderr' # ── (6) determinism ────────────────────────────────────────────────────────────────────────────────── "$BIN" corpus --skipped --max-file-size=1K --exclude=vendorgen --no-cache > "$TMP/sk2.xml" 2>/dev/null @@ -165,30 +170,35 @@ else fi # ── (8) built-in subtree prunes are counted, and counted apart from the user ones ──────────────────── -# pruned/: one indexable .cpp at the top, three subtrees the CRAWL prunes by policy (node_modules and -# dist from kCrawlSkipDirs, buildout/ via the CMakeCache.txt build-output sentinel) and one the USER -# prunes (--exclude=genstuff). The two classes must land in two different counters. -mkdir -p "$TMP/pruned/node_modules/pkg" "$TMP/pruned/dist" "$TMP/pruned/buildout" "$TMP/pruned/genstuff" +# pruned/: one indexable .cpp at the top, five subtrees the CRAWL prunes by policy (node_modules, dist, +# worktrees and .worktrees-clean from kCrawlSkipDirs, buildout/ via the CMakeCache.txt build-output +# sentinel) and one the USER prunes (--exclude=genstuff). The two classes must land in two different counters. +mkdir -p "$TMP/pruned/node_modules/pkg" "$TMP/pruned/dist" "$TMP/pruned/worktrees/nested" \ + "$TMP/pruned/.worktrees-clean/nested" "$TMP/pruned/buildout" "$TMP/pruned/genstuff" printf 'int prunedKeep( void ) { return 1; }\n' > "$TMP/pruned/keep.cpp" printf 'int nodeThing( void ) { return 2; }\n' > "$TMP/pruned/node_modules/pkg/m.cpp" printf 'int distThing( void ) { return 3; }\n' > "$TMP/pruned/dist/gen.cpp" +printf 'int worktreeThing( void ) { return 4; }\n' > "$TMP/pruned/worktrees/nested/w.cpp" +printf 'int cleanWorktreeThing( void ) { return 5; }\n' > "$TMP/pruned/.worktrees-clean/nested/w.cpp" printf '# CMake cache stub\n' > "$TMP/pruned/buildout/CMakeCache.txt" -printf 'int builtThing( void ) { return 4; }\n' > "$TMP/pruned/buildout/obj.cpp" -printf 'int genThing( void ) { return 5; }\n' > "$TMP/pruned/genstuff/g.cpp" +printf 'int builtThing( void ) { return 6; }\n' > "$TMP/pruned/buildout/obj.cpp" +printf 'int genThing( void ) { return 7; }\n' > "$TMP/pruned/genstuff/g.cpp" # (8a) presence guard — the fixture really holds a file under each pruned subtree -[ -f "$TMP/pruned/node_modules/pkg/m.cpp" ] && [ -f "$TMP/pruned/dist/gen.cpp" ] && [ -f "$TMP/pruned/buildout/obj.cpp" ] \ - && ok "(8) fixture has a source file under each of the 3 built-in-pruned subtrees" \ +[ -f "$TMP/pruned/node_modules/pkg/m.cpp" ] && [ -f "$TMP/pruned/dist/gen.cpp" ] \ + && [ -f "$TMP/pruned/worktrees/nested/w.cpp" ] && [ -f "$TMP/pruned/.worktrees-clean/nested/w.cpp" ] \ + && [ -f "$TMP/pruned/buildout/obj.cpp" ] \ + && ok "(8) fixture has a source file under each of the 5 built-in-pruned subtrees" \ || no "(8) fixture is missing a pruned-subtree source file — the arms below would pass by finding nothing" "$BIN" pruned --skipped --no-cache > "$TMP/prune_plain.xml" 2>/dev/null PR_PLAIN="$( attr "$TMP/prune_plain.xml" pruned_dirs )" EX_PLAIN="$( attr "$TMP/prune_plain.xml" excluded_dirs )" echo " (8) no --exclude: pruned_dirs=\"$PR_PLAIN\" excluded_dirs=\"$EX_PLAIN\"" -if [ -n "$PR_PLAIN" ] && [ "$PR_PLAIN" -ge 3 ] 2>/dev/null; then - ok "(8) built-in prunes are COUNTED — pruned_dirs=$PR_PLAIN (node_modules, dist, the CMakeCache sentinel)" +if [ -n "$PR_PLAIN" ] && [ "$PR_PLAIN" -ge 5 ] 2>/dev/null; then + ok "(8) built-in prunes are COUNTED — pruned_dirs=$PR_PLAIN (node_modules, dist, worktrees, .worktrees-clean, CMakeCache)" else - no "(8) pruned_dirs=\"${PR_PLAIN:-absent}\" — built-in subtree prunes are invisible (want >= 3)" + no "(8) pruned_dirs=\"${PR_PLAIN:-absent}\" — built-in subtree prunes are invisible (want >= 5)" fi [ "$EX_PLAIN" = "0" ] && ok "(8) excluded_dirs=0 with no --exclude — a policy prune is never miscounted as a user one" \ || no "(8) excluded_dirs=\"$EX_PLAIN\" with no --exclude given — want 0" From acbb53b6b880797f538f2eb7bc34e99a92b6a6c3 Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:38:25 -0300 Subject: [PATCH 07/13] fix(windows): keep latest main portable with clang-cl --- CMakeLists.txt | 10 +++++----- src/githarden.h | 4 ++++ src/wrap.h | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 561c594ec..0820a9863 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -508,7 +508,7 @@ set(RIPWIRE_SRCS # The global profile uses -ffast-math, but PageRank reductions must not reassociate (the determinism # contract's no-reassociation rule — docs/ARCHITECTURE.md §3). -if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") +if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "/fp:precise") else() set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "-fno-fast-math") @@ -559,8 +559,8 @@ target_link_libraries(ripwire_probe PRIVATE tree-sitter Threads::Threads) if(WIN32) target_link_libraries(ripwire_probe PRIVATE ws2_32) target_sources(ripwire_probe PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/win32/ripwire.manifest") - if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(ripwire_probe PRIVATE /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + target_compile_options(ripwire_probe PRIVATE /EHsc /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") else() target_compile_options(ripwire_probe PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") endif() @@ -584,8 +584,8 @@ target_link_libraries(ripwire PRIVATE tree-sitter Threads::Threads) if(WIN32) target_link_libraries(ripwire PRIVATE ws2_32) target_sources(ripwire PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/win32/ripwire.manifest") - if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(ripwire PRIVATE /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + target_compile_options(ripwire PRIVATE /EHsc /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") else() target_compile_options(ripwire PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") endif() diff --git a/src/githarden.h b/src/githarden.h index 40e5674a7..16b6716a8 100644 --- a/src/githarden.h +++ b/src/githarden.h @@ -210,7 +210,11 @@ inline bool appendGitConfigOverride( const char* key, const char* value ) const std::string keyN = "GIT_CONFIG_KEY_" + std::to_string( n ); const std::string valN = "GIT_CONFIG_VALUE_" + std::to_string( n ); const std::string count = std::to_string( n + 1 ); +#if defined(_WIN32) + return ::_putenv_s( keyN.c_str(), key ) == 0 && ::_putenv_s( valN.c_str(), value ) == 0 && ::_putenv_s( "GIT_CONFIG_COUNT", count.c_str() ) == 0; +#else return ::setenv( keyN.c_str(), key, 1 ) == 0 && ::setenv( valN.c_str(), value, 1 ) == 0 && ::setenv( "GIT_CONFIG_COUNT", count.c_str(), 1 ) == 0; +#endif } // ── the startup record, kept so --doctor reports the SAME probe main() acted on ───────────────────────────── diff --git a/src/wrap.h b/src/wrap.h index 77b2c9eab..ee70ecf3c 100644 --- a/src/wrap.h +++ b/src/wrap.h @@ -354,7 +354,7 @@ inline std::string wrapCommandToken( const std::string_view executablePath ) } std::error_code ec; const fs::path candidate = fs::path( std::string( dir ) ) / "ripwire"; - if( fs::is_regular_file( candidate, ec ) && !ec && ::access( candidate.c_str(), X_OK ) == 0 ) + if( fs::is_regular_file( candidate, ec ) && !ec && ::access( candidate.string().c_str(), X_OK ) == 0 ) { return "ripwire"; } From 173150b2eda9f2cbd8f96e05519a8926c85ca3b3 Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:21:46 -0300 Subject: [PATCH 08/13] perf(windows): restore incremental cache hits --- src/arch.h | 8 ++++---- src/ingest_cache.h | 20 ++++++++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/arch.h b/src/arch.h index 0a5008500..8b9b36885 100644 --- a/src/arch.h +++ b/src/arch.h @@ -556,7 +556,7 @@ inline std::string_view relForHash( std::string_view path, std::string_view root { // 1) strip the ingest-root prefix if present (allow one optional trailing '/' on the root). std::string_view rootTrim = root; - while( rootTrim.size() > 1 && rootTrim.back() == '/' ) + while( rootTrim.size() > 1 && ( rootTrim.back() == '/' || rootTrim.back() == '\\' ) ) { rootTrim.remove_suffix( 1 ); // "/abs/repo/" → "/abs/repo" } @@ -566,14 +566,14 @@ inline std::string_view relForHash( std::string_view path, std::string_view root // matched the root; the next char (if any) must be a '/' so we strip whole path components only // ("/abs/repo" must not eat the "repo" in "/abs/repository/..."). std::string_view rest = path.substr( rootTrim.size() ); - if( rest.empty() || rest.front() == '/' ) + if( rest.empty() || rest.front() == '/' || rest.front() == '\\' ) { path = rest; } } // 2) normalize residual leading "./" then leading "/" so "." / "./x" / "/x" all collapse to "x". - while( path.size() >= 2 && path[0] == '.' && path[1] == '/' ) + while( path.size() >= 2 && path[0] == '.' && ( path[1] == '/' || path[1] == '\\' ) ) { path.remove_prefix( 2 ); } @@ -581,7 +581,7 @@ inline std::string_view relForHash( std::string_view path, std::string_view root { path.remove_prefix( 1 ); } - while( path.size() >= 2 && path[0] == '.' && path[1] == '/' ) + while( path.size() >= 2 && path[0] == '.' && ( path[1] == '/' || path[1] == '\\' ) ) { path.remove_prefix( 2 ); } diff --git a/src/ingest_cache.h b/src/ingest_cache.h index 7252895df..67ce653bd 100644 --- a/src/ingest_cache.h +++ b/src/ingest_cache.h @@ -206,7 +206,7 @@ constexpr std::uint32_t kCacheVersion = 18; // 18: #62 — call refs i // (Py `pkg.mod`, TS `./x`, Rust `crate::a::b`/`mod:x`) — // a target FORMAT change → old caches must be rejected. // 4: Include gained a `bool isAngle` (quote/angle) field -constexpr std::uint32_t kParserVer = 88; // bump on any grammar/.scm/extraction change +constexpr std::uint32_t kParserVer = 89; // bump on any grammar/.scm/extraction or cache-key normalization change // 88 = 2026-09-10 (Dart, test/dartcheck.sh): a 23rd grammar joins // kLangTable, so the CRAWL ADMITS FILES IT PREVIOUSLY REFUSED — // a v87 blob has no record for the `.dart` it never saw, so the @@ -1598,7 +1598,12 @@ inline RawRouteUse readRouteUse( ByteR& r ) { RawRouteUse u; u.startByte = r.u32 inline std::string reAbsolutize( std::string_view rel, std::string_view root ) { std::string_view rootTrim = root; - while( rootTrim.size() > 1 && rootTrim.back() == '/' ) +#if defined( _WIN32 ) + constexpr char separator = '\\'; +#else + constexpr char separator = '/'; +#endif + while( rootTrim.size() > 1 && ( rootTrim.back() == '/' || rootTrim.back() == '\\' ) ) { rootTrim.remove_suffix( 1 ); } @@ -1609,8 +1614,15 @@ inline std::string reAbsolutize( std::string_view rel, std::string_view root ) std::string out; out.reserve( rootTrim.size() + 1 + rel.size() ); out.append( rootTrim ); - out.push_back( '/' ); - out.append( rel ); + out.push_back( separator ); + while( !rel.empty() && ( rel.front() == '/' || rel.front() == '\\' ) ) + { + rel.remove_prefix( 1 ); + } + for( const char c : rel ) + { + out.push_back( c == '/' || c == '\\' ? separator : c ); + } return out; } From 6f55b77797e53838e7c0026a77cf3d9a316dd77d Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:50:45 -0300 Subject: [PATCH 09/13] perf: cache extracted document text --- src/ingest_docpass.h | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/ingest_docpass.h b/src/ingest_docpass.h index 4f0257a4d..b9bc83003 100644 --- a/src/ingest_docpass.h +++ b/src/ingest_docpass.h @@ -25,39 +25,50 @@ namespace rw // function of the file BYTES, so bridge results are cached under the shared cache dir keyed by // content hash; the "ripwire-" prefix keeps eviction inside the existing family sweep (whose // age+size caps also bound a markitdown UPGRADE's staleness — the input-bytes key alone would never -// notice one). An EMPTY extraction is never cached: "" means markitdown absent or errored — a fact -// about the machine, not the bytes. Hand-rolled kinds (ipynb/html/csv) stay uncached (microseconds); -// cacheEnabled=false (--no-cache) bypasses the sidecar entirely. tmpKey keeps concurrent workers' +// notice one). Hand-rolled extractors use the same content-hash sidecar with a parser-version prefix. +// An EMPTY extraction is never cached: "" means markitdown absent or errored — a fact about the machine, +// not the bytes. cacheEnabled=false (--no-cache) bypasses the sidecars entirely. tmpKey keeps concurrent workers' // unpublished temp files distinct; the publish itself is a whole-file rename, so a concurrent // reader sees every byte or none. inline std::string docTextViaBridgeCache( const std::string& path, const std::string& ext, bool cacheEnabled, std::uint32_t tmpKey ) { std::string text; - std::string bridgeBlobPath; - if( cacheEnabled && docparse::docKindOf( ext ) == docparse::DocKind::Markitdown ) + std::string textBlobPath; + const docparse::DocKind kind = docparse::docKindOf( ext ); + if( cacheEnabled && kind != docparse::DocKind::None ) { std::string docBytes; if( docparse::detail::readWholeFile( path, docBytes ) ) { char blobName[ 64 ]; - rw::formatTo( blobName, sizeof( blobName ), "ripwire-docmd-{:016x}.bin", - static_cast( fnv1a64( docBytes ) ) ); - bridgeBlobPath = quality::resolveCacheBlobPath( quality::cacheDirLadder(), blobName ); - docparse::detail::readWholeFile( bridgeBlobPath, text ); // miss ⇒ text stays empty + if( kind == docparse::DocKind::Markitdown ) + { + rw::formatTo( blobName, sizeof( blobName ), "ripwire-docmd-{:016x}.bin", + static_cast( fnv1a64( docBytes ) ) ); + } + else + { + // Hand-rolled extraction is also pure, but its parser is part of the cache identity. + // Bump this when the ipynb/html/csv text shape changes; stale text is worse than a miss. + rw::formatTo( blobName, sizeof( blobName ), "ripwire-doctxt-1-{:016x}.bin", + static_cast( fnv1a64( docBytes ) ) ); + } + textBlobPath = quality::resolveCacheBlobPath( quality::cacheDirLadder(), blobName ); + docparse::detail::readWholeFile( textBlobPath, text ); // miss ⇒ text stays empty } } if( text.empty() ) { text = docparse::parseDocFile( path, ext ); - if( !text.empty() && !bridgeBlobPath.empty() ) + if( !text.empty() && !textBlobPath.empty() ) { - const std::string tmp = bridgeBlobPath + ".tmp" + std::to_string( tmpKey ); + const std::string tmp = textBlobPath + ".tmp" + std::to_string( tmpKey ); std::FILE* fp = std::fopen( tmp.c_str(), "wb" ); if( fp != nullptr ) { const bool wroteAll = std::fwrite( text.data(), 1, text.size(), fp ) == text.size(); std::fclose( fp ); - if( !wroteAll || std::rename( tmp.c_str(), bridgeBlobPath.c_str() ) != 0 ) + if( !wroteAll || std::rename( tmp.c_str(), textBlobPath.c_str() ) != 0 ) { std::remove( tmp.c_str() ); } From 9124d68928532d78e3d28c29ea52b8f42020e27a Mon Sep 17 00:00:00 2001 From: lennix1337 <28357644+lennix1337@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:07:29 -0300 Subject: [PATCH 10/13] port native Windows build and test harness --- .gitattributes | 4 + .github/workflows/ci.yml | 28 +- CMakeLists.txt | 87 +++- bench/agentloop/grade_answers.py | 27 +- bench/agentloop/run_agentloop.py | 19 +- bench/capsweep/capsweep.py | 22 +- cmake/PortableFlags.cmake | 10 + docs/TUNING.md | 74 +-- scripts/cxxstd.sh | 8 +- skills/install.sh | 27 +- src/arch.h | 19 +- src/ccjson.h | 2 +- src/cli.h | 27 + src/clones.h | 4 +- src/codexdoctor.h | 74 ++- src/crossref.h | 19 +- src/darkflags.h | 2 +- src/docdrift.h | 4 +- src/docparse.h | 3 +- src/editplan.h | 75 ++- src/editpreview.h | 16 +- src/githarden.h | 10 +- src/gitmine.h | 109 +++- src/gitoracle.h | 10 +- src/graph.h | 34 ++ src/graphlegend.h | 4 +- src/infra/jsonesc.h | 26 +- src/infra/platform_compat.cpp | 591 +++++++++++++++++++-- src/infra/platform_compat.h | 216 +++++++- src/infra/processLock.h | 10 +- src/infra/text.h | 28 + src/ingest.cpp | 16 +- src/ingest.h | 16 +- src/ingest_astquery.h | 55 +- src/ingest_cache.h | 13 +- src/ingest_crawl.h | 54 +- src/ingest_docpass.h | 28 +- src/ingest_parsepool.h | 6 +- src/ingest_prewarm.h | 6 +- src/lexical.h | 2 +- src/lintrules.h | 2 +- src/main.cpp | 82 +-- src/mcp.h | 48 +- src/mcpedit.h | 90 +++- src/mcpindex.h | 27 +- src/mcpverbs.h | 12 +- src/mention.h | 16 +- src/mergescout.h | 195 ++++++- src/naminglens.h | 2 +- src/packtask.h | 2 +- src/pathguard.h | 81 ++- src/pincensus.h | 2 +- src/quality.h | 147 ++++-- src/recall.h | 6 +- src/renamemine.h | 4 - src/resolve.h | 84 ++- src/sarif.h | 7 + src/scip.h | 25 + src/search.h | 11 +- src/serialize.h | 48 +- src/skillscan.h | 10 +- src/tracelocus.h | 6 +- src/verbs_change.h | 10 +- src/verbs_doctor.h | 14 +- src/verbs_lint.h | 4 +- src/verbs_navigate.h | 4 +- src/verbs_quality.h | 2 +- src/verbs_report.h | 13 +- src/workspace.h | 10 +- src/wrap.h | 20 +- test/adaptivecutshapecheck.sh | 67 ++- test/agentloopclaudecheck.sh | 6 +- test/agentloopeditsuitecheck.sh | 16 +- test/agentloopgradercheck.sh | 27 +- test/agentloopopencodecheck.sh | 14 +- test/agenttablecheck.sh | 4 +- test/anchorcheck.sh | 12 +- test/archmetricscheck.sh | 16 +- test/astqueryregexcheck.sh | 18 +- test/binoverridecheck.sh | 23 +- test/bodydialectcheck.sh | 13 +- test/budgetpolicycheck.sh | 58 ++- test/cachefuzzcheck.sh | 23 +- test/cacheisolationcheck.sh | 25 +- test/cacheisolationcheck_windows.py | 170 +++++++ test/callformcheck.sh | 6 +- test/capdisclosurecheck.sh | 6 +- test/capsweepcheck.sh | 5 +- test/childwalkscalecheck.sh | 4 + test/childwalktime_windows.py | 69 +++ test/churnjoincheck.sh | 21 +- test/clonecachecheck.sh | 42 +- test/clonededupcheck.sh | 6 + test/codexdoctorcheck.sh | 11 + test/codexdoctorcheck_windows.py | 345 +++++++++++++ test/compactlegendcheck.sh | 34 +- test/connectcorecheck.sh | 7 +- test/crawlescapecheck.sh | 52 +- test/crossrefcheck.sh | 2 +- test/deckcheck_allowlist.txt | 1 + test/decltodefcheck.sh | 119 ++++- test/defaultceilingcheck.sh | 19 +- test/docdemotecheck.sh | 33 +- test/editpayloadbinarycheck.sh | 15 +- test/editplanpayloadconfinecheck.sh | 15 +- test/editroundtripcheck.sh | 9 +- test/emitescapecheck.sh | 9 +- test/emptycorpuscheck.sh | 10 +- test/evictioncheck.sh | 46 +- test/fieldidcheck.sh | 12 +- test/forrootlegendcheck.sh | 50 ++ test/g1configcheck.sh | 19 +- test/gateexitcheck.sh | 51 +- test/headbinstagecheck.sh | 17 +- test/hostilecheck.sh | 66 ++- test/infraportcheck.sh | 15 +- test/installer_isolation.py | 41 +- test/lib/headbinlib.sh | 54 +- test/manifestcheck.sh | 21 +- test/maxfilesizecheck.sh | 12 +- test/mcpeditcheck.sh | 6 +- test/mcpeditmodecheck.sh | 20 +- test/mcpeditracecheck.sh | 180 ++++++- test/mcpreadloopcheck.sh | 33 +- test/mcpreloadcheck.sh | 10 +- test/mcpstalecheck.sh | 18 +- test/mcpstalecheck_windows.py | 138 +++++ test/mdsectioncheck.sh | 4 +- test/nestedimportcheck.sh | 37 +- test/noaliascheck.sh | 26 +- test/padscalecheck.sh | 8 +- test/pargates.py | 761 +++++++++++++++++++++++++++- test/pargatescheck.sh | 304 ++++++++++- test/portablebuildcheck.sh | 2 +- test/preproccondcheck.sh | 37 +- test/preprocdeadscalecheck.sh | 14 +- test/printf_parity.manifest | 2 +- test/printffmtparitycheck.sh | 6 +- test/probecheck.sh | 6 +- test/process_cpu.py | 106 ++++ test/qsnapprefetchcheck.sh | 25 + test/qsnapprefetchcheck_windows.py | 454 +++++++++++++++++ test/qualitysignalcheck.sh | 2 +- test/qualitystalecheck.sh | 60 ++- test/recallbufcheck.sh | 2 +- test/regression.sh | 175 ++++++- test/releaseinstallcheck.sh | 14 +- test/sidecarsymlinkcheck.sh | 16 +- test/sidecarsymlinkcheck_windows.py | 439 ++++++++++++++++ test/skillinstallcheck.sh | 54 +- test/skillscanreadcheck.sh | 17 +- test/sourceinstallcheck.sh | 9 +- test/strkerncheck.sh | 12 +- test/taskechocheck.sh | 21 +- test/tokenbudgetcheck.sh | 1 + test/utf8scrubcheck.sh | 18 +- test/worktreeleakcheck.sh | 211 ++++++-- test/wrapverbscheck.sh | 11 +- test/xmlcheck.py | 73 +++ 159 files changed, 6985 insertions(+), 852 deletions(-) create mode 100644 src/infra/text.h create mode 100644 test/cacheisolationcheck_windows.py create mode 100644 test/childwalktime_windows.py create mode 100644 test/codexdoctorcheck_windows.py create mode 100644 test/mcpstalecheck_windows.py create mode 100644 test/process_cpu.py create mode 100644 test/qsnapprefetchcheck_windows.py create mode 100644 test/sidecarsymlinkcheck_windows.py create mode 100644 test/xmlcheck.py diff --git a/.gitattributes b/.gitattributes index 7d770bec6..89c0276ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,3 +15,7 @@ # unusually well, and shrinking those headers to win a byte count would destroy the most useful thing in # the files. test/*.sh linguist-detectable=false + +# Frozen evaluation packs are length-prefixed binary artifacts; Git must not rewrite their LF bytes on Windows. +bench/recalleval/*.mdpack -text +bench/recalleval/*.srcpack -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5050d3021..d42e9c457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -476,12 +476,34 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Configure (portable — clang + ninja) - run: cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ + shell: bash + run: cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DRIPWIRE_LTO=OFF - name: Build run: cmake --build build -j + # Build the HEAD comparison binary once, before the suite. The monotonicity gates use it in staged + # mode; building it inside a gate would multiply the Windows compile cost and can race the gate budget. + - name: Stage the HEAD comparison binary (once before the Windows suite) + shell: bash + run: | + export TMPDIR="$( cygpath -u "$RUNNER_TEMP" )" + export RIPWIRE_HEADBIN_BUILD_LOG="$TMPDIR/headbin-build.log" + . test/lib/headbinlib.sh + hb="$( ripwire_head_binary "$PWD" "$TMPDIR" )" || { echo "HEAD binary build failed; last 80 lines of its log:"; tail -n 80 "$RIPWIRE_HEADBIN_BUILD_LOG"; exit 1; } + ripwire_headbin_verify "$hb" "$( git rev-parse HEAD )" + "$hb" --version + echo "RIPWIRE_HEADBIN=$hb" >> "$GITHUB_ENV" + - name: Doctor check run: .\build\ripwire.exe . --doctor @@ -499,6 +521,10 @@ jobs: shell: bash run: bash test/det-gate.sh build/ripwire.exe + - name: Full native Windows gate suite + shell: bash + run: python test/pargates.py . build/ripwire.exe -j 6 --budget-scale 4 + asan: name: asan (${{ matrix.os }}) strategy: diff --git a/CMakeLists.txt b/CMakeLists.txt index 933307227..fb0207771 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -695,6 +695,18 @@ if(RIPWIRE_TESTS) target_compile_definitions(ripwire_test_strkern PRIVATE RIPWIRE_TEST_ROOT="${CMAKE_CURRENT_SOURCE_DIR}") target_link_libraries(ripwire_test_strkern PRIVATE doctest::doctest) add_test(NAME ripwire.strkern COMMAND ripwire_test_strkern) + + if(WIN32) + foreach(_ripwire_test_target ripwire_test_csr ripwire_test_pagerank ripwire_test_radix ripwire_test_strkern) + target_sources(${_ripwire_test_target} PRIVATE src/infra/platform_compat.cpp) + target_link_libraries(${_ripwire_test_target} PRIVATE ws2_32) + if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + target_compile_options(${_ripwire_test_target} PRIVATE /EHsc /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + else() + target_compile_options(${_ripwire_test_target} PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h") + endif() + endforeach() + endif() endif() # ---- self-profiling build (src/infra/profileScope.h): -DRIPWIRE_PROFILE=ON ---- @@ -886,14 +898,45 @@ file(WRITE "${_ripwire_libstdcxx_ignorelist}" "src:*/include/c\\+\\+/*/print\n" "[implicit-unsigned-integer-truncation]\nsrc:*/include/c\\+\\+/*/format\n" "src:*/include/c\\+\\+/*/print\n") +if(WIN32) + # MSVC's filesystem prefix helper intentionally subtracts the lower bound from an unsigned packed value; + # Clang's integer sanitizer diagnoses that defined range test before any project code runs. Keep the + # exemption limited to the vendor header; project arithmetic remains covered by the complete G1 stack. + file(APPEND "${_ripwire_libstdcxx_ignorelist}" + "[unsigned-integer-overflow]\nsrc:*/include/filesystem\n" + "src:*filesystem\n" + "fun:*_Is_drive_prefix*\n") +endif() if(RIPWIRE_ASAN) + set(_ripwire_windows_clangcl_asan OFF) + set(_ripwire_no_omit_frame_pointer -fno-omit-frame-pointer) + # CMake's MSVC linker rule invokes CMAKE_LINKER directly. With ClangCL, passing the LLVM + # -fsanitize flags to link.exe instruments the objects but never loads the Clang sanitizer + # runtime, leaving __asan_* unresolved. Use clang-cl as the link driver for this one flavour; + # its driver preserves the MSVC ABI/import libraries and adds the matching runtime. + if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + set(_ripwire_windows_clangcl_asan ON) + set(_ripwire_no_omit_frame_pointer /Oy-) + string(REPLACE "" "" CMAKE_CXX_LINK_EXECUTABLE + "${CMAKE_CXX_LINK_EXECUTABLE}") + # The replacement template does not carry the generator's per-target FLAGS. Keep the + # sanitizer thunk and the instrumented objects on the same dynamic CRT as the compile rule. + string(REPLACE " ${CMAKE_CL_NOLOGO} " + " ${CMAKE_CL_NOLOGO} /MD ${RIPWIRE_G1_SANITIZERS}" + CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE}") + string(REPLACE " /out:" " /link /out:" + CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE}") + message(STATUS "RIPWIRE_ASAN: using the ClangCL driver for Windows sanitizer links") + endif() foreach(_t IN LISTS RIPWIRE_RUNTIME_COMPILE_TARGETS) target_compile_options(${_t} PRIVATE ${RIPWIRE_G1_SANITIZERS} - -fno-sanitize-recover=all -fno-omit-frame-pointer -O2 -g) + -fno-sanitize-recover=all ${_ripwire_no_omit_frame_pointer} -O2 -g) endforeach() foreach(_t IN LISTS RIPWIRE_RUNTIME_LINK_TARGETS) - target_link_options(${_t} PRIVATE ${RIPWIRE_G1_SANITIZERS}) + if(NOT _ripwire_windows_clangcl_asan) + target_link_options(${_t} PRIVATE ${RIPWIRE_G1_SANITIZERS}) + endif() endforeach() # Every exemption below names a check inside the Clang-only `integer` group, or uses # -fsanitize-ignorelist=, which GCC does not implement. With `integer` absent there is nothing to @@ -911,15 +954,47 @@ if(RIPWIRE_ASAN) endforeach() endif() - # Xcode's arm64 Darwin runtime rejects both the standalone leak sanitizer and detect_leaks=1 at startup. - # Keep the local ASan/UBSan gate executable and explicit; LeakSanitizer remains a Linux/upstream-runtime - # CI gate using the committed tree-sitter suppressions rather than being falsely claimed on Apple Clang. - if(APPLE) + # Darwin and Windows runtimes reject detect_leaks=1 at startup. Keep the local ASan/UBSan gate + # executable and explicit; LeakSanitizer remains a Linux/upstream-runtime CI gate using the + # committed tree-sitter suppressions rather than being falsely claimed on either platform. + if(APPLE OR WIN32) set(_ripwire_asan_options "detect_leaks=0:halt_on_error=1:abort_on_error=1") else() set(_ripwire_asan_options "detect_leaks=1:halt_on_error=1:abort_on_error=1") endif() + set(_ripwire_asan_runtime_copy_command) + if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + # clang-cl links the ASAN dynamic runtime, but Windows does not search Clang's resource + # directory for DLLs. Discover the matching directory from the active compiler so the fixture + # exercises this build without requiring a system-wide DLL installation. Copying beside the + # executable is deliberate: putting a semicolon-separated Windows PATH in a CMake command is + # parsed as a list of unrelated arguments by the Ninja generator. + execute_process( + COMMAND "${CMAKE_CXX_COMPILER}" -print-resource-dir + OUTPUT_VARIABLE _ripwire_clang_resource_dir + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + if(_ripwire_clang_resource_dir) + file(TO_CMAKE_PATH "${_ripwire_clang_resource_dir}/lib/windows" _ripwire_asan_runtime_dir) + list(APPEND _ripwire_asan_runtime_copy_command + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${_ripwire_asan_runtime_dir}/clang_rt.asan_dynamic-x86_64.dll" + "$") + message(STATUS "RIPWIRE_ASAN: fixture copies the runtime DLL from ${_ripwire_asan_runtime_dir}") + else() + message(WARNING "RIPWIRE_ASAN: clang resource directory was not discoverable; runtime fixture may miss the ASAN DLL") + endif() + endif() + if(WIN32 AND _ripwire_asan_runtime_dir) + foreach(_t IN LISTS RIPWIRE_RUNTIME_LINK_TARGETS) + add_custom_command(TARGET ${_t} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${_ripwire_asan_runtime_dir}/clang_rt.asan_dynamic-x86_64.dll" + "$") + endforeach() + endif() add_custom_target(ripwire_asan_fixture + ${_ripwire_asan_runtime_copy_command} COMMAND ${CMAKE_COMMAND} -E env "ASAN_OPTIONS=${_ripwire_asan_options}" "UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1" diff --git a/bench/agentloop/grade_answers.py b/bench/agentloop/grade_answers.py index d14e1cb61..a93fbc30c 100644 --- a/bench/agentloop/grade_answers.py +++ b/bench/agentloop/grade_answers.py @@ -156,10 +156,25 @@ def globstar_shell(): probed once and a row that NEEDS `**` is REFUSED when no capable shell exists, never guessed at.""" if not _SHELL: _SHELL.append( None ) - for candidate in ( "bash", "/opt/homebrew/bin/bash", "/usr/local/bin/bash", "/bin/bash" ): + if os.name == "nt": + configured = os.environ.get( "RIPWIRE_BASH", "" ) + candidates = [ configured, r"C:\\Program Files\\Git\\usr\\bin\\bash.exe", + r"C:\\Program Files (x86)\\Git\\usr\\bin\\bash.exe" ] + candidates = [ c for c in candidates if c and os.path.isfile( c ) ] + else: + candidates = ( "bash", "/opt/homebrew/bin/bash", "/usr/local/bin/bash", "/bin/bash" ) + for candidate in candidates: + if os.name == "nt": + normalized = os.path.normcase( os.path.abspath( candidate ) ) + if "\\windows\\system32\\" in normalized or "\\windowsapps\\" in normalized: + continue + probe_env = None + if os.name == "nt": + probe_env = os.environ.copy() + probe_env[ "PATH" ] = "/usr/bin:/bin:" + probe_env.get( "PATH", "" ) try: probe = subprocess.run( [ candidate, "-O", "globstar", "-c", "true" ], - capture_output=True, text=True, timeout=30 ) + capture_output=True, text=True, timeout=30, env=probe_env ) except ( OSError, subprocess.SubprocessError ): continue if probe.returncode == 0: @@ -170,9 +185,15 @@ def globstar_shell(): def run_gt( gt_command, pin_root, timeout_s=300 ): """Execute the derivation command at the pin, under bash (never the operator's zsh — §4).""" shell = globstar_shell() + if os.name == "nt" and shell is None: + return "", "no supported Git Bash executable found", 127 argv = ( [ shell, "-O", "globstar", "-O", "nullglob", "-c", gt_command ] if shell else [ "bash", "-c", gt_command ] ) - proc = subprocess.run( argv, capture_output=True, text=True, cwd=str( pin_root ), timeout=timeout_s ) + run_env = None + if os.name == "nt": + run_env = os.environ.copy() + run_env[ "PATH" ] = "/usr/bin:/bin:" + run_env.get( "PATH", "" ) + proc = subprocess.run( argv, capture_output=True, text=True, cwd=str( pin_root ), timeout=timeout_s, env=run_env ) return proc.stdout, proc.stderr, proc.returncode def derive_key( stdout ): diff --git a/bench/agentloop/run_agentloop.py b/bench/agentloop/run_agentloop.py index 030aac9f7..d50476c14 100644 --- a/bench/agentloop/run_agentloop.py +++ b/bench/agentloop/run_agentloop.py @@ -198,9 +198,14 @@ def sh( args, cwd=None, timeout=1800, env=None ): real cwd. A run launched from this checkout therefore had the agent's bash in the task repo and its read/glob/edit tools in THIS repository — the suite's first live run edited the committed fixture through that split. Set explicitly, once, here, for every harness (a shell started by codex or claude inherits - the same variable).""" + same variable).""" if cwd is not None: - env = dict( env if env is not None else os.environ, PWD=str( cwd ) ) + shell_pwd = str( cwd ) + if os.name == "nt": + drive, tail = os.path.splitdrive( os.path.abspath( shell_pwd ) ) + if drive: + shell_pwd = "/" + drive[ 0 ].lower() + tail.replace( "\\", "/" ) + env = dict( env if env is not None else os.environ, PWD=shell_pwd ) return subprocess.run( args, capture_output=True, text=True, timeout=timeout, cwd=cwd, env=env ) def checkout_repo( repo, base_commit, repos_dir ): @@ -375,10 +380,18 @@ def install_ripwire_shim( run_home, ripwire_bin ): log = shim_dir / "ripwire-calls.log" shim = shim_dir / "ripwire" real = str( pathlib.Path( ripwire_bin ).resolve() ) if os.sep in str( ripwire_bin ) else str( ripwire_bin ) + if os.name == "nt": + # The wrapper is a Git-Bash script even when the harness itself is native Python. Keep paths in + # slash form inside that script: a backslash in a single-quoted Bash path is literal, not a Windows + # separator, and would otherwise turn the log/real-binary path into a different filename. + real = real.replace( "\\", "/" ) + log_for_shell = str( log ).replace( "\\", "/" ) + else: + log_for_shell = str( log ) shim.write_text( "#!/usr/bin/env bash\n" "# generated by run_agentloop.py — logs argv, then execs the real binary unchanged.\n" - f"printf '%s\\n' \"$*\" >> {shlex.quote( str( log ) )}\n" + f"printf '%s\\n' \"$*\" >> {shlex.quote( log_for_shell )}\n" f"exec {shlex.quote( real )} \"$@\"\n" ) shim.chmod( 0o755 ) return str( shim ), log diff --git a/bench/capsweep/capsweep.py b/bench/capsweep/capsweep.py index 8d9c1ba70..a7a3b8ee0 100644 --- a/bench/capsweep/capsweep.py +++ b/bench/capsweep/capsweep.py @@ -135,6 +135,26 @@ def build(root, jobs): r = subprocess.run(['cmake', '--build', str(root / 'build'), '-j', str(jobs)], capture_output=True, text=True) return r.returncode, c.stdout + c.stderr + r.stdout + r.stderr +def binary_argv(binary): + """Return an argv prefix that can execute a native binary or a POSIX shebang stub on Windows.""" + binary = str(binary) + if os.name != 'nt': + return [binary] + try: + with open(binary, 'rb') as f: + is_script = f.read(2) == b'#!' + except OSError: + is_script = False + if not is_script: + return [binary] + bash = os.environ.get('RIPWIRE_BASH') or shutil.which('bash.exe') or shutil.which('bash') + if not bash: + raise RuntimeError('Windows run-corpus needs Git Bash to execute its shebang stub') + normalized = os.path.normcase(os.path.abspath(bash)).replace('/', '\\') + if '\\windows\\system32\\' in normalized or '\\windowsapps\\' in normalized: + raise RuntimeError('Windows run-corpus refuses the WSL bash launcher; set RIPWIRE_BASH to Git Bash') + return [bash, binary] + def offenders(log, known): """Cap names the compiler rejected as non-constant — they must stay constexpr.""" bad = set() @@ -269,7 +289,7 @@ def run_corpus(binary, root, corpus, env, timeout=120): sizes[line], states[line] = None, '%s: $%s' % (kStateUnexpanded, missingVar.args[0]) continue try: - r = subprocess.run([str(binary)] + argv + ['--no-cache'], cwd=str(root), + r = subprocess.run(binary_argv(binary) + argv + ['--no-cache'], cwd=str(root), capture_output=True, stdin=subprocess.DEVNULL, env=e, timeout=timeout) except subprocess.TimeoutExpired: sizes[line], states[line] = None, kStateTimeout # recorded, never silently dropped diff --git a/cmake/PortableFlags.cmake b/cmake/PortableFlags.cmake index 6c8e729a7..82d7aecfb 100644 --- a/cmake/PortableFlags.cmake +++ b/cmake/PortableFlags.cmake @@ -44,6 +44,12 @@ option(RIPWIRE_PRETEND_LINUX # clang >= 17 (AppleClang 16, the Xcode 16.2 the release pins), and on clang 16 a baseline x86-64 binary # running strkern.h's scalar twins. test/portablebuildcheck.sh #2d-#2g hold both directions. set(RIPWIRE_TARGET_ARCH "${CMAKE_SYSTEM_PROCESSOR}") +if(NOT RIPWIRE_TARGET_ARCH AND DEFINED CMAKE_CXX_COMPILER_ARCHITECTURE_ID) + set(RIPWIRE_TARGET_ARCH "${CMAKE_CXX_COMPILER_ARCHITECTURE_ID}") +endif() +if(RIPWIRE_TARGET_ARCH MATCHES "^(x64|X64|amd64|AMD64)$") + set(RIPWIRE_TARGET_ARCH "x86_64") +endif() if(APPLE AND CMAKE_OSX_ARCHITECTURES) list(LENGTH CMAKE_OSX_ARCHITECTURES _ripwire_osx_arch_count) if(_ripwire_osx_arch_count EQUAL 1) @@ -80,6 +86,10 @@ if(RIPWIRE_NATIVE) else() set(RIPWIRE_ARCH_FLAGS -O3 -march=native -ffast-math -fno-finite-math-only) endif() +elseif(MSVC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND RIPWIRE_IS_X86_64) + # ClangCL accepts the MSVC frontend flags but still needs the LLVM architecture level explicitly; + # keeping this branch ahead of the generic MSVC one is what enables strkern.h's AVX2 path on Windows. + set(RIPWIRE_ARCH_FLAGS /O2 /clang:-march=x86-64-v3 /fp:precise /permissive- /utf-8) elseif(MSVC) # MSVC compiler flags: precise math (preserves isnan/isfinite), conformant C++ mode, UTF-8 source/exec charset set(RIPWIRE_ARCH_FLAGS /O2 /fp:precise /permissive- /utf-8) diff --git a/docs/TUNING.md b/docs/TUNING.md index de2807fe8..f5d705be6 100644 --- a/docs/TUNING.md +++ b/docs/TUNING.md @@ -60,7 +60,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kForLensDefaultTopN` = `40` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `320` — **12 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `320` — **12 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -79,7 +79,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kSpecificMinLen` = `8` -`src/graph.h` — discloses: `importers_capped` — probe value `64` — **12 verb(s) respond** +`src\graph.h` — discloses: `importers_capped` — probe value `64` — **12 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -98,7 +98,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kDocMentionMaxAnchors` = `8` -`src/mention.h` — discloses: `doc_mentions_capped`, `mention_files_capped`, `mention_syms_capped`, `mention_tokens_capped` — probe value `64` — **11 verb(s) respond** +`src\mention.h` — discloses: `doc_mentions_capped`, `mention_files_capped`, `mention_syms_capped`, `mention_tokens_capped` — probe value `64` — **11 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -116,7 +116,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kForFileTailShownCap` = `24` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `192` — **10 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `192` — **10 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -133,7 +133,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kForPayloadBudgetBytes` = `7500` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `60000` — **10 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `60000` — **10 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -150,7 +150,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kForCapTailSigBytes` = `96` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `768` — **8 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `768` — **8 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -165,7 +165,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kMaxExpandSibs` = `100` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `800` — **5 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `800` — **5 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -177,7 +177,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kCommonNameDefThreshold` = `5` -`src/graph.h` — discloses: `importers_capped` — probe value `40` — **4 verb(s) respond** +`src\graph.h` — discloses: `importers_capped` — probe value `40` — **4 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -188,7 +188,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kLintMaxPerRule` = `5000` -`src/lintrules.h` — discloses: **none** — probe value `40000` — **4 verb(s) respond** +`src\lintrules.h` — discloses: **none** — probe value `40000` — **4 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -199,7 +199,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kDocMentionMaxDocsPerAnchor` = `2` -`src/mention.h` — discloses: `doc_mentions_capped`, `mention_files_capped`, `mention_syms_capped`, `mention_tokens_capped` — probe value `34` — **3 verb(s) respond** +`src\mention.h` — discloses: `doc_mentions_capped`, `mention_files_capped`, `mention_syms_capped`, `mention_tokens_capped` — probe value `34` — **3 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -209,7 +209,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kExternalSurfaceRowCap` = `100` -`src/pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `800` — **2 verb(s) respond** +`src\pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `800` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -218,7 +218,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kForAutoBodyBudgetBytes` = `6000` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `48000` — **2 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `48000` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -227,7 +227,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kHandoffDocRows` = `4` -`src/handoff.h` — discloses: `syms_capped` — probe value `36` — **2 verb(s) respond** +`src\handoff.h` — discloses: `syms_capped` — probe value `36` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -236,7 +236,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kHandoffSymbolsPerCodeFile` = `50` -`src/handoff.h` — discloses: `syms_capped` — probe value `400` — **2 verb(s) respond** +`src\handoff.h` — discloses: `syms_capped` — probe value `400` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -245,7 +245,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kHandoffSymbolsPerDocFile` = `12` -`src/handoff.h` — discloses: `syms_capped` — probe value `96` — **2 verb(s) respond** +`src\handoff.h` — discloses: `syms_capped` — probe value `96` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -254,7 +254,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kOrdinalWindowCap` = `40` -`src/ensemble.h` — discloses: `files_capped`, `findings_capped`, `syms_capped` — probe value `320` — **2 verb(s) respond** +`src\ensemble.h` — discloses: `files_capped`, `findings_capped`, `syms_capped` — probe value `320` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -263,7 +263,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kPrDefaultBudgetTokens` = `8000` -`src/prcontext.h` — discloses: **none** — probe value `64000` — **2 verb(s) respond** +`src\prcontext.h` — discloses: **none** — probe value `64000` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -272,7 +272,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kUnitComplexityLowRiskMax` = `5` -`src/dmm.h` — discloses: **none** — probe value `40` — **2 verb(s) respond** +`src\dmm.h` — discloses: **none** — probe value `40` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -281,7 +281,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kUnitInterfacingLowRiskMax` = `2` -`src/dmm.h` — discloses: **none** — probe value `34` — **2 verb(s) respond** +`src\dmm.h` — discloses: **none** — probe value `34` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -290,7 +290,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kUnitSizeLowRiskMax` = `15` -`src/dmm.h` — discloses: **none** — probe value `120` — **2 verb(s) respond** +`src\dmm.h` — discloses: **none** — probe value `120` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -299,7 +299,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kZoomTopModuleCap` = `40` -`src/pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `320` — **2 verb(s) respond** +`src\pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `320` — **2 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -308,7 +308,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kBatchCap` = `16` -`src/mcpverbs.h` — discloses: `blast_radius_capped`, `coboost_commits_capped`, `forgotten_capped`, `hits_capped`, `siblings_capped`, `unindexed_candidates_capped` — probe value `128` — **1 verb(s) respond** +`src\mcpverbs.h` — discloses: `blast_radius_capped`, `coboost_commits_capped`, `forgotten_capped`, `hits_capped`, `siblings_capped`, `unindexed_candidates_capped` — probe value `128` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -316,7 +316,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kCallHierarchyRowCap` = `40` -`src/pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `320` — **1 verb(s) respond** +`src\pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `320` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -324,7 +324,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kCellsPerRowCap` = `12` -`src/nonlocalstate.h` — discloses: `cells_capped`, `decls_capped` — probe value `96` — **1 verb(s) respond** +`src\nonlocalstate.h` — discloses: `cells_capped`, `decls_capped` — probe value `96` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -332,7 +332,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kDefaultRecallMaxTokens` = `8000` -`src/recall.h` — discloses: **none** — probe value `64000` — **1 verb(s) respond** +`src\recall.h` — discloses: **none** — probe value `64000` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -340,7 +340,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kDefsPerNameCap` = `8` -`src/contextratio.h` — discloses: `defs_capped`, `files_capped`, `syms_capped` — probe value `64` — **1 verb(s) respond** +`src\contextratio.h` — discloses: `defs_capped`, `files_capped`, `syms_capped` — probe value `64` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -348,7 +348,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kEnsembleFileRowCap` = `20` -`src/ensemble.h` — discloses: `files_capped`, `findings_capped`, `syms_capped` — probe value `160` — **1 verb(s) respond** +`src\ensemble.h` — discloses: `files_capped`, `findings_capped`, `syms_capped` — probe value `160` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -356,7 +356,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kFileRowCap` = `40` -`src/contextratio.h` — discloses: `defs_capped`, `files_capped`, `syms_capped` — probe value `320` — **1 verb(s) respond** +`src\contextratio.h` — discloses: `defs_capped`, `files_capped`, `syms_capped` — probe value `320` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -364,7 +364,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kGrepMatchedLineMaxBytes` = `512` -`src/search.h` — discloses: `hits_capped` — probe value `4096` — **1 verb(s) respond** +`src\search.h` — discloses: `hits_capped` — probe value `4096` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -372,7 +372,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kMaxExpandIncludes` = `24` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `192` — **1 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `192` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -380,7 +380,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kMentionMaxSymbolsPerFile` = `3` -`src/mention.h` — discloses: `doc_mentions_capped`, `mention_files_capped`, `mention_syms_capped`, `mention_tokens_capped` — probe value `35` — **1 verb(s) respond** +`src\mention.h` — discloses: `doc_mentions_capped`, `mention_files_capped`, `mention_syms_capped`, `mention_tokens_capped` — probe value `35` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -388,7 +388,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kPanelRowCap` = `40` -`src/qualitypanel.h` — discloses: `findings_capped` — probe value `320` — **1 verb(s) respond** +`src\qualitypanel.h` — discloses: `findings_capped` — probe value `320` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -396,7 +396,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kSituBlastFilesShown` = `8` -`src/situ.h` — discloses: `tests_capped`, `untested_capped` — probe value `64` — **1 verb(s) respond** +`src\situ.h` — discloses: `tests_capped`, `untested_capped` — probe value `64` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -404,7 +404,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kSliceFlowDefaultDepth` = `8` -`src/slice.h` — discloses: **none** — probe value `64` — **1 verb(s) respond** +`src\slice.h` — discloses: **none** — probe value `64` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -412,7 +412,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kSymbolRowCap` = `40` -`src/contextratio.h` — discloses: `defs_capped`, `files_capped`, `syms_capped` — probe value `320` — **1 verb(s) respond** +`src\contextratio.h` — discloses: `defs_capped`, `files_capped`, `syms_capped` — probe value `320` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -420,7 +420,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kTreeRowCap` = `80` -`src/pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `640` — **1 verb(s) respond** +`src\pageview.h` — discloses: `count_capped`, `findings_capped`, `hits_capped`, `importers_capped`, `modules_capped` — probe value `640` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | @@ -428,7 +428,7 @@ moving. Re-run `prepare|screen|sweep` to refresh them. ### `kWithGraphNodeCap` = `8` -`src/serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `64` — **1 verb(s) respond** +`src\serialize.h` — discloses: `calls_capped`, `inc_capped`, `sibs_capped` — probe value `64` — **1 verb(s) respond** | invocation | default | at probe | delta | | --- | --- | --- | --- | diff --git a/scripts/cxxstd.sh b/scripts/cxxstd.sh index c97c91a75..bab993c7d 100644 --- a/scripts/cxxstd.sh +++ b/scripts/cxxstd.sh @@ -40,12 +40,16 @@ # ripwire_cxx_std_flag CXX → prints the C++23 flag spelling CXX accepts; rc=1 if it accepts neither. ripwire_cxx_std_flag() { - local cxx="${1:-c++}" probe flag rc=1 + local cxx="${1:-c++}" probe probe_arg flag rc=1 probe="$( mktemp -d )" printf 'int main(){ return 0; }\n' > "$probe/cxxstd_probe.cpp" + probe_arg="$probe/cxxstd_probe.cpp" + case "$( uname -s 2>/dev/null )" in + MINGW*|MSYS*) probe_arg="$( cygpath -w "$probe_arg" )" ;; + esac for flag in -std=c++23 -std=c++2b; do - if "$cxx" "$flag" -fsyntax-only "$probe/cxxstd_probe.cpp" >/dev/null 2>&1; then + if "$cxx" "$flag" -fsyntax-only "$probe_arg" >/dev/null 2>&1; then rc=0 break fi diff --git a/skills/install.sh b/skills/install.sh index 5b9ba938f..b0114271d 100755 --- a/skills/install.sh +++ b/skills/install.sh @@ -13,8 +13,29 @@ set -eu src="$( cd "$( dirname "$0" )" && pwd )" +# Git Bash's `ln -sfn` can materialize a directory-like MSYS link that native +# Windows tools cannot identify or prune. Use a real directory symlink on +# Windows, while keeping the POSIX installer path unchanged. +isWindowsShell=0 +case "${OSTYPE:-}" in + msys*|cygwin*|mingw*) isWindowsShell=1 ;; +esac +link_skill() +{ + if [ "$isWindowsShell" -eq 1 ] && command -v cygpath >/dev/null 2>&1 && command -v cmd.exe >/dev/null 2>&1; then + targetNative="$( cygpath -w "$1" )" + destNative="$( cygpath -w "$2" )" + if [ -e "$2" ] || [ -L "$2" ]; then + [ -L "$2" ] || { echo "skills/install.sh: refusing to replace a real directory at $2" >&2; return 1; } + rm -f "$2" + fi + MSYS_NO_PATHCONV=1 cmd.exe /d /c mklink /D "$destNative" "$targetNative" >/dev/null 2>&1 + else + ln -sfn "$1" "$2" + fi +} + # ── the PreToolUse matcher, in one place. It is not cosmetic: a matcher decides which tool calls the -# hook is ever SHOWN. Read/Glob are here because the whole-file read is the largest token sink in an # agent loop and the one default a skill description cannot intercept; mcp__ripwire__.* is here for # the hook's other job, the substitution meter (docs/SUBSTITUTION_METER.md), whose numerator would # otherwise miss every agent that prefers the MCP server to the CLI; Edit/Write/MultiEdit/ @@ -350,7 +371,7 @@ for d in "$src"/ripwire-*/; do skipped=$(( skipped + 1 )) continue fi - ln -sfn "$d" "$dst/$name" + link_skill "$d" "$dst/$name" echo "installed $name -> $dst/$name" count=$(( count + 1 )) done @@ -375,7 +396,7 @@ if [ "$mode" = "hermes" ]; then skipped=$(( skipped + 1 )) continue fi - ln -sfn "$nd" "$dst/$nname" + link_skill "$nd" "$dst/$nname" echo "installed $nname -> $dst/$nname (Hermes-native skill)" count=$(( count + 1 )) done diff --git a/src/arch.h b/src/arch.h index fe2563fa3..11194ce32 100644 --- a/src/arch.h +++ b/src/arch.h @@ -556,14 +556,29 @@ inline std::uint64_t fnv1a64( std::string_view s ) noexcept // than an empty one. Empty root ⇒ just the leading-`./`/`/` normalization (equivalent to root "."). inline std::string_view relForHash( std::string_view path, std::string_view root ) noexcept { + const auto samePathChar = []( char a, char b ) noexcept + { + if( a == '\\' ) { a = '/'; } + if( b == '\\' ) { b = '/'; } +#if defined( _WIN32 ) + if( a >= 'A' && a <= 'Z' ) { a = char( a - 'A' + 'a' ); } + if( b >= 'A' && b <= 'Z' ) { b = char( b - 'A' + 'a' ); } +#endif + return a == b; + }; + // 1) strip the ingest-root prefix if present (allow one optional trailing '/' on the root). std::string_view rootTrim = root; while( rootTrim.size() > 1 && ( rootTrim.back() == '/' || rootTrim.back() == '\\' ) ) { rootTrim.remove_suffix( 1 ); // "/abs/repo/" → "/abs/repo" } - if( !rootTrim.empty() && rootTrim != "." && path.size() >= rootTrim.size() - && path.compare( 0, rootTrim.size(), rootTrim ) == 0 ) + bool rootMatches = !rootTrim.empty() && rootTrim != "." && path.size() >= rootTrim.size(); + for( std::size_t i = 0; rootMatches && i < rootTrim.size(); ++i ) + { + rootMatches = samePathChar( path[i], rootTrim[i] ); + } + if( rootMatches ) { // matched the root; the next char (if any) must be a '/' so we strip whole path components only // ("/abs/repo" must not eat the "repo" in "/abs/repository/..."). diff --git a/src/ccjson.h b/src/ccjson.h index a25cadac4..2e12357a2 100644 --- a/src/ccjson.h +++ b/src/ccjson.h @@ -54,7 +54,7 @@ struct CcFileMetrics // Degrades to 0 if the file cannot be opened (deleted between crawl and export). inline std::uint32_t ccCountLoc( const std::string& path ) noexcept { - std::FILE* fp = std::fopen( path.c_str(), "rb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( path.c_str(), "rb" ); if( !fp ) { return 0; diff --git a/src/cli.h b/src/cli.h index 6cc863543..0e5402bca 100644 --- a/src/cli.h +++ b/src/cli.h @@ -3297,6 +3297,26 @@ static_assert( std::size( kBoolFlags ) + std::size( kViewFlags ) + std::size( kI // flag matched, and did its value survive" is one question with one answer. enum class ViewFlagMatch : std::uint8_t { NoMatch, Assigned, Refused }; +inline constexpr std::string_view kPathValuePrefixes[] = +{ + "--eval-mined=", "--eval-skills=", "--arch=", "--cache=", "--index-out=", "--scip=", "--pin-census=", + "--lint-rules=", "--exercises=", "--cochange=", "--situ=", "--test-gate=", "--scan-skills=", "--dead-code=", + "--plan-lint=", "--scan-skill=", "--batch=", "--at=", "--edit-payload=", "--edit-target-file=", "--edit-plan=", + "--eval-stray=", "--from-trace=", "--with-profile=", "--brief=", "--html=", "--affected=" +}; + +inline bool isPathValuePrefix( std::string_view prefix ) noexcept +{ + for( const std::string_view pathPrefix : kPathValuePrefixes ) + { + if( prefix == pathPrefix ) + { + return true; + } + } + return false; +} + inline ViewFlagMatch applyViewFlag( std::string_view arg, Config& c ) { for( const ViewFlag& vf : kViewFlags ) @@ -3306,6 +3326,13 @@ inline ViewFlagMatch applyViewFlag( std::string_view arg, Config& c ) continue; } const std::string_view value = arg.substr( vf.prefix.size() ); +#if defined( _WIN32 ) + if( isPathValuePrefix( vf.prefix ) ) + { + // argv storage is mutable and Config deliberately borrows it as a view. + rw::compat::rw_normalize_msys_drive_paths_in_place( const_cast( value.data() ) ); + } +#endif // §B5: the EMPTY-value decision is the row's, never this loop's. Refuse prints here; Meaningful and // HandlerRefuses both fall through to the assignment — the difference between them is which code // OWNS the refusal, and the row records it (the consteval floor beside the table pins the columns). diff --git a/src/clones.h b/src/clones.h index 605146fb4..0fe599c22 100644 --- a/src/clones.h +++ b/src/clones.h @@ -350,7 +350,7 @@ inline std::vector findClones( const IngestResult& ing, int minToken { continue; } - std::FILE* fp = std::fopen( diskPath( ing, std::uint32_t( f ) ).c_str(), "rb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( diskPath( ing, std::uint32_t( f ) ).c_str(), "rb" ); if( !fp ) { continue; @@ -647,7 +647,7 @@ inline std::vector findClonesType3( const IngestResult& ing, int min { continue; } - std::FILE* fp = std::fopen( diskPath( ing, std::uint32_t( f ) ).c_str(), "rb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( diskPath( ing, std::uint32_t( f ) ).c_str(), "rb" ); if( !fp ) { continue; // degrade: unreadable file just contributes no candidates (never a crash) diff --git a/src/codexdoctor.h b/src/codexdoctor.h index 36b3f3bf4..ac15a4c07 100644 --- a/src/codexdoctor.h +++ b/src/codexdoctor.h @@ -5,7 +5,9 @@ // emitted: a doctor report may be pasted into an issue, so unrelated tokens and secrets stay dark. #include +#include "infra/platform_compat.h" #include +#include #include #include #include @@ -45,13 +47,70 @@ inline std::string readSmallFile( const std::filesystem::path& path, bool& ok ) return text; } +#if defined( _WIN32 ) || defined( _MSC_VER ) +inline bool windowsExecutableFile( const std::string& path ) +{ + const std::wstring widePath = rw::compat::rw_utf8_to_wide( rw::compat::rw_windows_path_from_msys( path ) ); + if( widePath.empty() ) { return false; } + const DWORD attributes = ::GetFileAttributesW( widePath.c_str() ); + if( attributes == INVALID_FILE_ATTRIBUTES || ( attributes & FILE_ATTRIBUTE_DIRECTORY ) != 0 ) { return false; } + const HANDLE handle = ::CreateFileW( widePath.c_str(), FILE_EXECUTE | FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr ); + if( handle == INVALID_HANDLE_VALUE ) { return false; } + ::CloseHandle( handle ); + return true; +} + +inline std::string windowsExecutableCandidate( const std::string& path ) +{ + const std::string native = rw::compat::rw_windows_path_from_msys( path ); + if( windowsExecutableFile( native ) ) { return native; } + const std::size_t separator = native.find_last_of( "\\/" ); + const std::size_t dot = native.find( '.', separator == std::string::npos ? 0 : separator + 1 ); + if( dot == std::string::npos && windowsExecutableFile( native + ".exe" ) ) { return native + ".exe"; } + return {}; +} +#endif + inline std::string resolveExecutable( std::string_view command ) { if( command.empty() ) { return {}; } - const auto executable = []( const std::string& path ) +#if defined( _WIN32 ) || defined( _MSC_VER ) + const auto hasPath = command.find( '/' ) != std::string_view::npos || command.find( '\\' ) != std::string_view::npos + || ( command.size() >= 2 && command[ 1 ] == ':' ); + if( hasPath ) { return windowsExecutableCandidate( std::string( command ) ); } + const char* pathEnv = std::getenv( "PATH" ); + const std::string_view pathList = pathEnv ? std::string_view( pathEnv ) : std::string_view(); + const bool semicolonList = pathList.find( ';' ) != std::string_view::npos; + for( std::size_t at = 0; at <= pathList.size(); ) { - return ::access( path.c_str(), X_OK ) == 0; - }; + std::size_t split = pathList.size(); + if( semicolonList ) + { + const std::size_t found = pathList.find( ';', at ); + if( found != std::string_view::npos ) { split = found; } + } + else + { + for( std::size_t i = at; i < pathList.size(); ++i ) + { + const bool driveColon = i == at + 1 && i > 0 + && ( ( pathList[ at ] >= 'A' && pathList[ at ] <= 'Z' ) + || ( pathList[ at ] >= 'a' && pathList[ at ] <= 'z' ) ); + if( pathList[ i ] == ':' && !driveColon ) { split = i; break; } + } + } + const std::string_view dir = pathList.substr( at, split - at ); + const std::string candidate = std::string( dir.empty() ? "." : dir ) + "/" + std::string( command ); + const std::string resolved = windowsExecutableCandidate( candidate ); + if( !resolved.empty() ) { return resolved; } + if( split == pathList.size() ) { break; } + at = split + 1; + } + return {}; +#else + const auto executable = []( const std::string& path ) { return ::access( path.c_str(), X_OK ) == 0; }; if( command.find( '/' ) != std::string_view::npos ) { const std::string path( command ); @@ -69,6 +128,7 @@ inline std::string resolveExecutable( std::string_view command ) remaining.remove_prefix( split + 1 ); } return {}; +#endif } inline Check binaryCheck( const std::string& selfPath ) @@ -78,6 +138,7 @@ inline Check binaryCheck( const std::string& selfPath ) struct stat activeSt {}; const bool haveSelf = !selfPath.empty() && ::stat( selfPath.c_str(), &selfSt ) == 0; const bool haveActive = !active.empty() && ::stat( active.c_str(), &activeSt ) == 0; + const bool same = haveSelf && haveActive && selfSt.st_dev == activeSt.st_dev && selfSt.st_ino == activeSt.st_ino; const bool copied = haveSelf && haveActive && selfSt.st_mtime == activeSt.st_mtime && selfSt.st_size == activeSt.st_size; // `copied` is a HEURISTIC pass (mtime+size equality, the cp -p install shape) — it cannot prove byte @@ -310,8 +371,11 @@ inline std::vector inspect( const std::string& selfPath ) const std::string home = envOr( "HOME", "" ); const std::filesystem::path agentHome = envOr( "AGENTS_HOME", home + "/.agents" ); const std::filesystem::path codexHome = envOr( "CODEX_HOME", home + "/.codex" ); - return { binaryCheck( selfPath ), skillsCheck( agentHome / "skills" ), hooksCheck( codexHome / "hooks.json" ), - mcpCheck( codexHome / "config.toml" ) }; + const Check binary = binaryCheck( selfPath ); + const Check skills = skillsCheck( agentHome / "skills" ); + const Check hooks = hooksCheck( codexHome / "hooks.json" ); + const Check mcp = mcpCheck( codexHome / "config.toml" ); + return { binary, skills, hooks, mcp }; } } // namespace rw::codexdoctor diff --git a/src/crossref.h b/src/crossref.h index 66c9dc629..be0615935 100644 --- a/src/crossref.h +++ b/src/crossref.h @@ -479,7 +479,7 @@ inline void streamBlobs( const std::string& root, const std::vector const std::string listPath = quality::cacheDirLadder() + "/ripwire-crossref-" + std::to_string( ::getpid() ) + ".shas"; { - std::FILE* lf = std::fopen( listPath.c_str(), "wb" ); + std::FILE* lf = rw::compat::rw_fopen_utf8( listPath.c_str(), "wb" ); if( !lf ) { st.startFailed = true; @@ -493,13 +493,8 @@ inline void streamBlobs( const std::string& root, const std::vector std::fclose( lf ); } -#ifdef _WIN32 - const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) - + " cat-file --batch < " + rw_short_path( listPath ) + " 2>/dev/null"; -#else const std::string cmd = "git -c core.quotepath=false -C " + shSingleQuote( root ) + " cat-file --batch < " + shSingleQuote( listPath ) + " 2>/dev/null"; -#endif std::FILE* pipe = popen( cmd.c_str(), "r" ); if( !pipe ) { @@ -699,11 +694,7 @@ struct RefInfo inline std::vector enumerateRefs( const std::string& root, std::string_view filter, const std::string& headSha, std::size_t* filterNameHits = nullptr ) { -#ifdef _WIN32 - const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format=^%(refname:short^)^|^%(objectname^)^|^%(committerdate:short^) refs/heads 2>/dev/null" ); -#else const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format='%(refname:short)|%(objectname)|%(committerdate:short)' refs/heads 2>/dev/null" ); -#endif std::vector out; for( std::string_view line : splitLines( raw ) ) { @@ -825,11 +816,7 @@ inline void parallelIndexed( std::size_t count, Body body ) return; } - std::size_t hwThreadCount = std::thread::hardware_concurrency(); - if( hwThreadCount == 0 ) - { - hwThreadCount = 1; - } + const std::size_t hwThreadCount = rw::compat::rw_effective_hardware_concurrency(); const std::size_t workerCount = std::min( { hwThreadCount, count, kMaxGitWorkers } ); if( workerCount <= 1 ) { @@ -1789,7 +1776,7 @@ inline EvalReport evalStray( const std::string& root, const std::string& labelsP std::string bytes; { - std::FILE* fp = std::fopen( labelsPath.c_str(), "rb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( labelsPath.c_str(), "rb" ); if( !fp ) { rep.ok = false; return rep; } char buf[ 65536 ]; std::size_t n = 0; diff --git a/src/darkflags.h b/src/darkflags.h index 42391b312..2880e3d3e 100644 --- a/src/darkflags.h +++ b/src/darkflags.h @@ -745,7 +745,7 @@ inline FileHarvest harvestFile( std::string_view bytes, std::string_view path, b // through keeps what was read, and an empty file is an engaged empty string. inline std::optional readWhole( const std::string& path ) { - std::FILE* fp = std::fopen( path.c_str(), "rb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( path.c_str(), "rb" ); if( !fp ) { return std::nullopt; diff --git a/src/docdrift.h b/src/docdrift.h index ef34125b0..a13cee935 100644 --- a/src/docdrift.h +++ b/src/docdrift.h @@ -2034,7 +2034,7 @@ inline void forEachIndexParallel( std::size_t count, const char* what, Work&& wo } }; - const std::size_t hwThreadCount = std::thread::hardware_concurrency(); + const std::size_t hwThreadCount = rw::compat::rw_effective_hardware_concurrency(); const std::size_t workerCount = std::min( { hwThreadCount ? hwThreadCount : 1, count, std::size_t( 16 ) } ); if( workerCount <= 1 ) { indexWorker(); return; } @@ -2153,7 +2153,7 @@ inline std::size_t scanCorpusFacts( const IngestResult& ing, const std::string& return 0; } - const std::size_t hwThreadCount = std::thread::hardware_concurrency(); + const std::size_t hwThreadCount = rw::compat::rw_effective_hardware_concurrency(); const std::size_t blockCount = std::min( { ( hwThreadCount ? hwThreadCount : 1 ) * 4, scanCount, std::size_t( 64 ) } ); const std::size_t blockSpan = ( scanCount + blockCount - 1 ) / blockCount; diff --git a/src/docparse.h b/src/docparse.h index 22a491dad..c570ffb3b 100644 --- a/src/docparse.h +++ b/src/docparse.h @@ -1,5 +1,6 @@ #pragma once #include "infra/emit.h" // rw::emitTo / emitRaw / formatTo — THE emitter and its siblings +#include "infra/platform_compat.h" // docparse.h — P1-B document ingest. Turns non-code documents that live IN a repo @@ -181,7 +182,7 @@ namespace detail // string, not a failure — a caller for which empty and unreadable mean the same thing says so with value_or. inline std::optional readWholeFile( const std::string& path ) { - std::FILE* fp = std::fopen( path.c_str(), "rb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( path.c_str(), "rb" ); if( fp == nullptr ) { return std::nullopt; diff --git a/src/editplan.h b/src/editplan.h index 6d37cad8b..9630060ce 100644 --- a/src/editplan.h +++ b/src/editplan.h @@ -3,6 +3,7 @@ #include "mcpedit.h" #include "nextverb.h" // E4: nextFlag — the shell-safe spelling of the rollback message's one next: +#include #include namespace rw::editplan @@ -54,21 +55,71 @@ struct FileStage std::vector edits; }; +inline bool isEditPathAbsolute( std::string_view path ) noexcept +{ + if( path.empty() || path.front() == '/' ) + { + return !path.empty(); + } +#if defined( _WIN32 ) + return ( path.front() == '\\' ) || ( path.size() >= 2 && path[ 1 ] == ':' ); +#else + return false; +#endif +} + +inline std::size_t lastEditPathSeparator( std::string_view path ) noexcept +{ +#if defined( _WIN32 ) + return path.find_last_of( "/\\" ); +#else + return path.find_last_of( '/' ); +#endif +} + inline std::string siblingPath( std::string_view planPath, std::string_view payload ) { - if( payload.empty() || payload.front() == '/' ) { return std::string( payload ); } - const std::size_t slash = planPath.find_last_of( '/' ); + if( payload.empty() || isEditPathAbsolute( payload ) ) { return std::string( payload ); } + const std::size_t slash = lastEditPathSeparator( planPath ); return slash == std::string_view::npos ? std::string( payload ) : std::string( planPath.substr( 0, slash + 1 ) ) + std::string( payload ); } +inline std::string canonicalEditPlanPath( std::string_view path ) +{ +#if defined( _WIN32 ) + const std::string native = rw::compat::rw_windows_path_from_msys( path ); + std::error_code ec; + const std::filesystem::path absolute = std::filesystem::absolute( std::filesystem::path( native ), ec ); + if( ec ) + { + return {}; + } + const std::filesystem::path canonical = std::filesystem::canonical( absolute, ec ); + if( !ec ) + { + return canonical.generic_string(); + } + ec.clear(); + const std::filesystem::path weak = std::filesystem::weakly_canonical( absolute, ec ); + return ec ? std::string() : weak.generic_string(); +#else + char buf[ PATH_MAX ]; + return ::realpath( std::string( path ).c_str(), buf ) != nullptr ? std::string( buf ) : std::string(); +#endif +} + // The directory a plan's payloads must live in: the plan file's own, canonicalized. "" when it cannot be // resolved, which the confinement check below treats as "cannot prove containment" and therefore refuses. inline std::string planDirAbs( const std::string& planPath ) { - const std::size_t slash = planPath.find_last_of( '/' ); + const std::size_t slash = lastEditPathSeparator( planPath ); const std::string dir = slash == std::string::npos ? std::string( "." ) : planPath.substr( 0, slash ); +#if defined( _WIN32 ) + return canonicalEditPlanPath( dir.empty() ? std::string_view( "." ) : std::string_view( dir ) ); +#else char buf[ PATH_MAX ]; return ::realpath( dir.empty() ? "/" : dir.c_str(), buf ) != nullptr ? std::string( buf ) : std::string(); +#endif } // A5: a plan's `payload` names a file whose BYTES are spliced into a source file, so an unconfined payload @@ -95,7 +146,7 @@ inline bool payloadWithinPlanDir( const std::string& planPath, const std::string // "../" payload, which is the exact bug this function exists to catch). char cwdBuf[ PATH_MAX ]; const std::string cwd = ::getcwd( cwdBuf, sizeof( cwdBuf ) ) != nullptr ? std::string( cwdBuf ) : std::string(); - if( cwd.empty() && payloadPath.front() != '/' ) + if( cwd.empty() && !isEditPathAbsolute( payloadPath ) ) { resolved = payloadPath; return false; // cannot place a relative path in any frame ⇒ cannot prove containment ⇒ refuse @@ -103,7 +154,7 @@ inline bool payloadWithinPlanDir( const std::string& planPath, const std::string // rw::lexicalNormalize (resolve.h) is the house's segment-stack `.`/`..` folder — the SAME primitive the // include resolver keys every path index through. It returns "" for a relative `..` that escapes above // its own base, which is already the answer this check wants. - const std::string lexical = lexicalNormalize( payloadPath.front() == '/' ? payloadPath : cwd + "/" + payloadPath ); + const std::string lexical = lexicalNormalize( isEditPathAbsolute( payloadPath ) ? payloadPath : cwd + "/" + payloadPath ); if( lexical.empty() ) { resolved = payloadPath; @@ -113,12 +164,26 @@ inline bool payloadWithinPlanDir( const std::string& planPath, const std::string // realpath is the AUTHORITY when the payload exists: `dir` is canonical, so only a canonical candidate // is comparable to it (a symlinked prefix such as /tmp -> /private/tmp otherwise reads as an escape), // and it is what catches a symlink sitting INSIDE the plan directory that points out of it. +#if defined( _WIN32 ) + const std::string canonical = canonicalEditPlanPath( lexical ); + if( !canonical.empty() ) + { + resolved = canonical; + return pathIsUnder( resolved, dir ); + } + if( rw::pathguard::isSymlink( lexical ) ) + { + resolved = lexical; + return false; + } +#else char buf[ PATH_MAX ]; if( ::realpath( lexical.c_str(), buf ) != nullptr ) { resolved = std::string( buf ); return pathIsUnder( resolved, dir ); } +#endif // The payload does not exist. realpath cannot speak, so judge it lexically: "../../../../etc/nope" must // still read as an escape rather than as a merely unreadable payload. resolved = lexical; diff --git a/src/editpreview.h b/src/editpreview.h index d3cf24dfd..75903fa1b 100644 --- a/src/editpreview.h +++ b/src/editpreview.h @@ -259,7 +259,7 @@ inline IngestResult ingestOneFile( const std::string& tmpDir, const std::string& std::error_code ec; const fs::path target = fs::path( tmpDir ) / fs::path( rel ); fs::create_directories( target.parent_path(), ec ); - std::FILE* fp = std::fopen( target.string().c_str(), "wb" ); + std::FILE* fp = rw::compat::rw_fopen_utf8( target.string().c_str(), "wb" ); if( fp == nullptr ) { DEGRADED_PATH_ALERT( "edit-preview: cannot write the spliced file into the temp root" ); @@ -278,15 +278,19 @@ inline IngestResult ingestOneFile( const std::string& tmpDir, const std::string& return ingest( tmpDir.c_str(), {}, {}, maxFileBytes, captureValueUses ); } -// E3: `CDATA` — src[a,b) as on disk, budgeted by WHOLE LINES: over -// kPreviewOverwriteBudgetBytes the CDATA is the head, with shown= its size, capped="1" and elided_lines= the rest. -// The CDATA goes through appendCdataSafe like every served body (a ]]> inside the span is split, never broken). +// E3: `CDATA` — the selected src[a,b) content, with the same presentation-only +// CRLF normalization as served bodies. `bytes=` is the normalized content size carried by the CDATA; over +// kPreviewOverwriteBudgetBytes the CDATA is the head by whole lines, with shown= its size, capped="1" and elided_lines= the rest. The CDATA goes +// through appendCdataSafe like every served body (a ]]> inside the span is split, never broken). inline constexpr std::size_t kPreviewOverwriteBudgetBytes = 4096; inline std::string overwriteChildXml( const std::string& src, std::size_t a, std::size_t b ) { - const std::string_view span = std::string_view( src ).substr( a, b - a ); - const mcpedit::LineRange lines = mcpedit::lineRangeOf( src, a, b ); + const std::string_view rawSpan = std::string_view( src ).substr( a, b - a ); + const mcpedit::LineRange lines = mcpedit::lineRangeOf( src, a, b ); + std::string spanText( rawSpan ); + normalizeCrlfInPlace( spanText ); + const std::string_view span = spanText; std::size_t shown = span.size(); std::uint32_t elidedLines = 0; if( span.size() > kPreviewOverwriteBudgetBytes ) diff --git a/src/githarden.h b/src/githarden.h index cb38de6ff..a0e84047f 100644 --- a/src/githarden.h +++ b/src/githarden.h @@ -33,6 +33,7 @@ #include "gitmine.h" // rw::popenTrimmed — the one popen-and-trim shape in the tree (never a second) #include "infra/emit.h" // rw::emitTo — the house emitter; no new printf-family site #include "infra/jsonesc.h" // rw::shSingleQuote +#include "infra/platform_compat.h" #include #include @@ -251,12 +252,17 @@ inline const Report& hardenForRoots( std::span roots ) bool anyHook = false; for( std::string_view root : roots ) { + const std::string rootStr = +#if defined( _WIN32 ) + rw::compat::rw_windows_path_from_msys( root ); +#else + std::string( root ); +#endif std::error_code ec; - if( root.empty() || !std::filesystem::is_directory( std::filesystem::path( root ), ec ) || ec ) + if( rootStr.empty() || !std::filesystem::is_directory( std::filesystem::path( rootStr ), ec ) || ec ) { continue; } - const std::string rootStr = std::string( root ); const FsmonitorForm form = localConfigMayCarryFsmonitor( rootStr ) ? probeFsmonitorForm( rootStr ) : FsmonitorForm::Unset; r.forms.emplace_back( rootStr, form ); anyHook = anyHook || form == FsmonitorForm::Hook; diff --git a/src/gitmine.h b/src/gitmine.h index e16d5835f..fb69dc045 100644 --- a/src/gitmine.h +++ b/src/gitmine.h @@ -16,6 +16,7 @@ #include "infra/stdinline.h" // readByteSafeLine — THE line reader (R4); no fixed buffer to split a long path on #include "infra/jsonesc.h" // A4-F27 residual: rw::shSingleQuote lives here (lightest shared header) — // gitmine.h no longer carries its own copy; see jsonesc.h for the dedup rationale +#include "infra/platform_compat.h" #include #include // the join's once-per-process disclosure flags @@ -26,6 +27,7 @@ #include #include #include +#include #include // gitRepoToplevel's per-directory memo — one rev-parse probe per root, not per miner #include #include @@ -513,6 +515,36 @@ inline bool isBoundarySuffix( std::string_view indexedPath, std::string_view git return indexedPath.compare( off, gitRelPath.size(), gitRelPath ) == 0 && ( off == 0 || indexedPath[ off - 1 ] == '/' ); } +inline bool gitPathPrefixMatches( std::string_view path, std::string_view prefix ) noexcept +{ + if( path.size() < prefix.size() ) + { + return false; + } + for( std::size_t i = 0; i < prefix.size(); ++i ) + { +#if defined( _WIN32 ) + const bool driveLetter = i == 0 && prefix.size() >= 2 && path.size() >= 2 && path[1] == ':' && prefix[1] == ':' + && ( ( path[0] >= 'A' && path[0] <= 'Z' ) || ( path[0] >= 'a' && path[0] <= 'z' ) ) + && ( ( prefix[0] >= 'A' && prefix[0] <= 'Z' ) || ( prefix[0] >= 'a' && prefix[0] <= 'z' ) ); + if( driveLetter ) + { + const char pathDrive = path[0] >= 'a' && path[0] <= 'z' ? char( path[0] - 'a' + 'A' ) : path[0]; + const char prefixDrive = prefix[0] >= 'a' && prefix[0] <= 'z' ? char( prefix[0] - 'a' + 'A' ) : prefix[0]; + if( pathDrive == prefixDrive ) + { + continue; + } + } +#endif + if( path[i] != prefix[i] ) + { + return false; + } + } + return true; +} + // ONE normalisation, applied to BOTH sides of the join before any byte comparison, and the only latitude the // join has. Two rewrites, in ONE pass so there is no second place to keep in step: // * every `/./` seam collapses — workspace.h spelled a merged-root file `