diff --git a/.ci/check-matrix-lists.sh b/.ci/check-matrix-lists.sh index e6139db0..e4d6addf 100755 --- a/.ci/check-matrix-lists.sh +++ b/.ci/check-matrix-lists.sh @@ -70,6 +70,7 @@ test-shim-urandom-wrap test-shim-futex-toctou test-ptrace-interrupt test-shim-cred-race +test-mmap-fastpath test-mremap-infra test-mremap-fork-tracking test-dev-shm-paths diff --git a/Makefile b/Makefile index 2ac3f79b..f5a216df 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,14 @@ $(ELFUSE_BIN): $(OBJS) | $(BUILD_DIR) # Native test binaries (macOS, Hypervisor.framework) +$(BUILD_DIR)/test-materialize-host: $(BUILD_DIR)/test-materialize-host.o \ + $(filter-out $(BUILD_DIR)/main.o,$(OBJS)) | $(BUILD_DIR) + $(call link-and-sign,$@,$^) + +$(BUILD_DIR)/test-pt-epoch-host: $(BUILD_DIR)/test-pt-epoch-host.o \ + $(filter-out $(BUILD_DIR)/main.o,$(OBJS)) | $(BUILD_DIR) + $(call link-and-sign,$@,$^) + ## Build the multi-vCPU HVF validation test (native macOS binary) $(BUILD_DIR)/test-multi-vcpu: $(BUILD_DIR)/test-multi-vcpu.o | $(BUILD_DIR) $(call link-and-sign,$@,$<) @@ -465,6 +473,33 @@ $(BUILD_DIR)/test-sigsuspend: tests/test-sigsuspend.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -Itests -o $@ $< -lpthread +# bench-mmap has a multi-threaded mmap_lock-contention section; needs -lpthread. +$(BUILD_DIR)/bench-mmap: tests/bench-mmap.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +# test-mmap-lazy races concurrent first touch from several threads. +$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +.PHONY: test-mmap-lazy +test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-lazy + +# EL1 consumer-mmap integration/stress test. +$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +.PHONY: test-mmap-fastpath +test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-fastpath + # test-thread-churn creates >64 threads to force thread-table slot reuse. $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" diff --git a/docs/internals.md b/docs/internals.md index 2ada3b61..058e2117 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -527,6 +527,18 @@ goes above the structured area, never below. Post-push masking ### `mmap` Notes +Private anonymous mappings are lazy at 2 MiB materialization granularity. A +host-side hierarchical bitmap records which low-VA 2 MiB blocks contain any +valid TTBR0 PTE, independently of the dirty-block bitmap. `munmap` and recycled +fast-path arenas use this index to visit only materialized blocks, so untouched +multi-GiB reservations have length-independent teardown. When every mapping in +a per-vCPU arena has been released and the index confirms that no PTE remains, +the arena cursor rewinds in place instead of taking a refill HVC. EL1 clears +the descriptors itself and advances a page-table epoch in shim_data; the host +translation cache checks that epoch alongside its own generation counter, so a +host thread that never takes `mmap_lock` stops translating the retired range at +once. + Aligned file-backed `MAP_SHARED` (fixed or non-fixed) installs a real host `mmap(MAP_FIXED|MAP_SHARED, fd)` overlay onto the guest slab so the kernel page cache keeps the mapping coherent with the file (and diff --git a/docs/usage.md b/docs/usage.md index c569e032..fc6c550e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -93,6 +93,16 @@ host `KEY=` imports as `KEY=`. An empty variable name is rejected. Given neither `--env` nor `--clear-env`, the guest inherits the host environment unchanged. `--clear-env` starts from nothing, leaving only what `--env` puts back. +### mmap call fast path + +The aarch64 EL1 consumer fast path is enabled by default for +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)` up to +32 GiB per request. +Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted +arenas, and full consumption rings fall back to the normal host syscall path. +Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host +path so observability and translated-guest behavior are unchanged. + ## Common Launch Patterns Run a statically linked guest binary: diff --git a/mk/shim.mk b/mk/shim.mk index 235c635e..7e167170 100644 --- a/mk/shim.mk +++ b/mk/shim.mk @@ -1,11 +1,33 @@ -# EL1 kernel shim assembly pipeline +# EL1 kernel shim pipeline # -# shim.S -> shim.o -> shim.bin -> shim_blob.h (C byte array) +# shim.S + freestanding shim-mmap.c -> shim.o -> shim.bin -> shim_blob.h -$(BUILD_DIR)/shim.o: src/core/shim.S | $(BUILD_DIR) +# Disable RCpc codegen so acquire loads remain cumulative LDARs. The retire +# snapshot rule carries cross-vCPU causality through different atomic words; +# LDAPR is intentionally too weak for that protocol. +SHIM_CFLAGS := -O2 -Wall -Wextra -Wpedantic -Wshadow \ + -Wstrict-prototypes -Wmissing-prototypes -Wformat=2 \ + -Wimplicit-fallthrough -Wundef -Wnull-dereference \ + -Wno-unused-parameter -ffreestanding -fno-builtin \ + -fno-stack-protector -fno-unwind-tables \ + -fno-asynchronous-unwind-tables -mno-outline-atomics \ + -Xclang -target-feature -Xclang -rcpc +SHIM_LD ?= ld + +$(BUILD_DIR)/shim-asm.o: src/core/shim.S | $(BUILD_DIR) @echo " AS $<" $(Q)$(SHIM_AS) $(SHIM_ASFLAGS) -o $@ $< +$(BUILD_DIR)/shim-mmap.o: src/core/shim-mmap.c src/core/shim-mmap.h \ + src/core/mmap-fastpath.h src/core/shim-globals.h | $(BUILD_DIR) + @echo " CC $<" + $(Q)$(CC) $(SHIM_CFLAGS) -MMD -MP -MF $(BUILD_DIR)/shim-mmap.d \ + -Isrc -c -o $@ $< + +$(BUILD_DIR)/shim.o: $(BUILD_DIR)/shim-asm.o $(BUILD_DIR)/shim-mmap.o + @echo " LD $@" + $(Q)$(SHIM_LD) -static -arch arm64 -e _start -o $@ $^ + $(BUILD_DIR)/shim.bin: $(BUILD_DIR)/shim.o @echo " OBJCOPY $@" $(Q)$(OBJCOPY) -O binary $< $@ diff --git a/mk/tests.mk b/mk/tests.mk index 724bcf03..7699c5e1 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -314,6 +314,8 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage check-eintr-contract ch $(call run-lane,test-shebang-host,shebang parser unit test) $(call run-lane,test-shim-futex-stats,futex EL1 fast path is live) $(call run-lane,test-gva-contracts,proved/gva.h call-site contract checks) + $(call run-lane,test-materialize-host,lazy materialization and signal-frame boundaries) + $(call run-lane,test-pt-epoch-host,EL1 PT epoch retires host translation cache entries) $(call run-lane,test-proctitle-host,proctitle argv-tail regression) $(call run-lane,test-proctitle-low-stack,proctitle low-stack regression) $(call run-lane,test-busybox,busybox applet validation) @@ -1380,6 +1382,14 @@ test-shim-futex-stats: $(ELFUSE_BIN) $(TEST_DIR)/test-shim-futex-fast \ $(TEST_DIR)/test-shim-futex-fast \ $(TEST_DIR)/test-futex-wake-nowaiter +.PHONY: test-materialize-host +test-materialize-host: $(BUILD_DIR)/test-materialize-host + @$(BUILD_DIR)/test-materialize-host + +.PHONY: test-pt-epoch-host +test-pt-epoch-host: $(BUILD_DIR)/test-pt-epoch-host + @$(BUILD_DIR)/test-pt-epoch-host + ## Run busybox applet smoke tests test-busybox: $(ELFUSE_BIN) $(BUSYBOX_DEPS) @if [ ! -x "$(BUSYBOX_BIN)" ]; then \ diff --git a/mk/verify.mk b/mk/verify.mk index 541fa0e0..deeeb6e0 100644 --- a/mk/verify.mk +++ b/mk/verify.mk @@ -468,6 +468,35 @@ VERIFY_FUTEXOP_MODEL := typed VERIFY_FUTEXOP_SCAN := src/proved/futexop.h VERIFY_FUTEXOP_CLAIM := for ANY guest-supplied val3 word VERIFY_FUTEXOP_UNPROVED := the wake and requeue walks around them stay test-covered +# Includes align.h: request_fits calls align_up_ok and window_fits, so this +# proof must discharge their contracts too, not merely assume them, the same +# reason VERIFY_ELF appends VERIFY_UTILS_FCTS. +# +# MIN_GOALS is the complete count from a real run. Keep the floor at that count +# so a future edit that quietly drops a contract or a runtime-error obligation +# cannot turn a smaller proof into a pass. +# +# mmap_fastpath_pow2_clamped was originally the classic bit-smear +# round-up-to-power-of-two; that form's bound and power-of-two properties +# were confirmed unreachable by these provers (a single OR step already +# times out), the same wall align_up_ok's own history describes one level +# down. Rewritten to a doubling loop with an axiomatized power-of-two ghost +# invariant -- same inputs, same outputs, linear arithmetic instead of +# bitwise -- and it discharges completely; see the comment above it. +VERIFY_MMAPFASTPATH_SRC := src/proved/mmap-fastpath.h +VERIFY_MMAPFASTPATH_FCTS := mmap_fastpath_request_fits mmap_fastpath_pow2_clamped \ + mmap_fastpath_window_max mmap_fastpath_arena_size \ + align_up_ok window_fits +VERIFY_MMAPFASTPATH_MIN_GOALS ?= 93 +VERIFY_MMAPFASTPATH_MODEL := typed +VERIFY_MMAPFASTPATH_SCAN := src/proved/mmap-fastpath.h src/proved/align.h +VERIFY_MMAPFASTPATH_CLAIM := for ANY cursor/limit/len and ANY registration history +VERIFY_MMAPFASTPATH_UNPROVED := mmap_fastpath_window_max reporting an actual array member \ + rather than just an upper bound (the loop-invariant \ + preservation step for that claim times out even with a \ + ghost witness index, confirmed unreachable, see the comment \ + above it); the atomic control-block bookkeeping around all \ + four stays test-covered VERIFY_FUTEXREQ_SRC := src/proved/futexreq.h VERIFY_FUTEXREQ_FCTS := futex_requeue_counts_valid futex_requeue_budget diff --git a/scripts/check-mutants.py b/scripts/check-mutants.py index e1c151af..e1126223 100755 --- a/scripts/check-mutants.py +++ b/scripts/check-mutants.py @@ -388,6 +388,41 @@ def _load(stem, name): " return start <= limit && length <= limit - start;", " return start <= limit && length <= limit - start + 1;", ), + # ---- verify-mmapfastpath ---------------------------------------------- + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_request_fits", + "accept every non-empty request (an allocation can pass the limit)", + " return window_fits(start, len, limit);", + " return true;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_pow2_clamped", + "clamp an oversized arena request to the lower bound", + " if (value >= MMAP_FAST_ARENA_MAX)\n" + " return MMAP_FAST_ARENA_MAX;", + " if (value >= MMAP_FAST_ARENA_MAX)\n" + " return MMAP_FAST_ARENA_MIN;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_window_max", + "discard a new maximum (the arena can be undersized)", + " if (window[i] > max)\n max = window[i];", + " if (window[i] > max)\n max = 0;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_arena_size", + "return an arena size above the configured maximum", + " return adaptive > covering ? adaptive : covering;", + " return UINT64_MAX;", + ), # ---- verify-cmsg ------------------------------------------------------- ( "cmsg", diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index 0e0d23a1..4fca2006 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -37,6 +37,7 @@ #include "syscall/signal.h" #include "debug/log.h" +#include "core/mmap-fastpath.h" /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter. @@ -138,6 +139,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len) guest_invalidate_ptes(g, 0, 0x1000); } +/* ELF/shim/stack bytes are populated directly in the slab before their page + * tables become live. Mark their semantic backing regardless of requested + * permissions: a read-only file segment is still nonzero and must be scrubbed + * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block. + * Synthetic page-table coverage for unallocated mmap space has no semantic + * region and therefore remains clean. + */ +static void mark_registered_backing_dirty(guest_t *g) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->end <= r->start) + continue; + uint64_t len = r->end - r->start; + if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base) + continue; + guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len); + } +} + int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info) { memset(info, 0, sizeof(*info)); @@ -302,7 +323,15 @@ static bool build_boot_regions(mem_region_t *regions, * to the vDSO page when splitting the block; otherwise vdso_build cannot * write into it through guest_ptr. */ - if (!append_boot_region(regions, nregions, g->shim_base, + + /* EL1 fast munmap walks and atomically clears the live TTBR0 tree. Give the + * page-table pool an identity VA visible only to EL1; EL0 remains unable to + * inspect or corrupt descriptors, and every guest syscall still rejects the + * encompassing infrastructure range. + */ + if (!append_boot_region(regions, nregions, g->pt_pool_base, g->pt_pool_end, + MEM_PERM_RW_EL1_ONLY) || + !append_boot_region(regions, nregions, g->shim_base, g->shim_base + shim_bin_len, MEM_PERM_RX) || /* shim_data is EL1-only: the guest must not directly read or write the @@ -568,6 +597,7 @@ int guest_bootstrap_prepare(guest_t *g, } register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); startup_trace_step("register_regions", t0); log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0, @@ -761,6 +791,13 @@ int guest_bootstrap_create_vcpu(guest_t *g, */ shim_globals_set_singleton(g); + /* Publish the main vCPU's first arena only after shim_globals_init has + * cleared every recycled control slot. Verbose tracing keeps all shim + * syscall fast paths on HVC so the trace remains complete. + */ + if (!verbose) + mmap_fastpath_prepare_vcpu(g, current_thread); + HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, CNTKCTL_EL1_EL0_TIMER_EN)); @@ -869,6 +906,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g, g->rosetta_guest_base - g->rosetta_va_base, ROSETTA_PATH); register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); int rosetta_argc = 0; const char **rosetta_argv = NULL; diff --git a/src/core/guest.c b/src/core/guest.c index af0d1c6b..5391648c 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -41,13 +41,15 @@ #include #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "proved/gva.h" #include "core/startup-trace.h" #include "debug/log.h" #include "utils.h" -#include "runtime/futex.h" /* futex_interrupt_request */ -#include "runtime/thread.h" /* thread_destroy_all_vcpus */ -#include "syscall/proc.h" /* proc_request_exit_group */ +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" /* thread_destroy_all_vcpus */ +#include "syscall/internal.h" /* mmap_lock (lazy fault-in) */ +#include "syscall/proc.h" /* proc_request_exit_group */ #include "syscall/signal.h" #include "syscall/wakeup-pipe.h" @@ -160,6 +162,7 @@ static const void *guest_host_memchr(const void *src, /* Forward declaration (defined in the page table section below) */ static int desc_to_perms(uint64_t desc); +static uint64_t *find_l2_entry(guest_t *g, uint64_t va); /* Page table pool allocator. */ @@ -356,6 +359,48 @@ static inline void pte_store_release(uint64_t *entry, uint64_t desc) memory_order_release); } +/* Low-VA TTBR0 occupancy index. All mutators run under mmap_lock (or during + * single-threaded bootstrap), so plain bitmap operations are sufficient. The + * page-table descriptors themselves remain release-published for lock-free + * guest walkers; this host-only index is never consulted by a vCPU. + */ +static inline bool guest_pte_present_index(uint64_t va, uint64_t *block_out) +{ + if (va >= GUEST_PTE_PRESENT_LIMIT) + return false; + *block_out = va / BLOCK_2MIB; + return true; +} + +static inline void guest_pte_present_set(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] |= 1ULL << (block & 63); + g->pte_present_summary[word >> 6] |= 1ULL << (word & 63); +} + +static inline void guest_pte_present_clear(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] &= ~(1ULL << (block & 63)); + if (g->pte_present_blocks[word] == 0) + g->pte_present_summary[word >> 6] &= ~(1ULL << (word & 63)); +} + +static inline bool guest_pte_present_test(const guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return false; + return (g->pte_present_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + /* Public API */ /* FEAT_TLBIRANGE probe -- runs exactly once via pthread_once. ARMv8.4 @@ -396,6 +441,24 @@ static void tlbi_range_probe_run(void) lse2_raw != 0; } +static int guest_init_materialize_cond(guest_t *g) +{ + int rc = pthread_cond_init(&g->materialize_cond, NULL); + if (rc == 0) + return 0; + log_error("guest: materialize condition initialization failed: %s", + strerror(rc)); + hv_vm_destroy(); + munmap(g->host_base, g->guest_size); + g->host_base = NULL; + g->n_segments = 0; + if (g->shm_fd >= 0) { + close(g->shm_fd); + g->shm_fd = -1; + } + return -1; +} + int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) { uint64_t t0; @@ -577,8 +640,7 @@ int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; - - return 0; + return guest_init_materialize_cond(g); } int guest_init_from_shm(guest_t *g, @@ -689,6 +751,8 @@ int guest_init_from_shm(guest_t *g, */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + if (guest_init_materialize_cond(g) < 0) + return -1; log_debug( "guest: CoW fork: mapped %llu GiB from shm " @@ -835,6 +899,7 @@ void guest_destroy(guest_t *g) close(g->shm_fd); g->shm_fd = -1; } + pthread_cond_destroy(&g->materialize_cond); } /* Check whether a candidate IPA range [gpa, gpa+size) overlaps the primary @@ -1264,9 +1329,11 @@ int guest_map_va_range(guest_t *g, * guest_split_block instead. Skip silently to mirror upstream's * sys_mmap_high_va "reuse existing GPA" behavior. */ + guest_pte_present_set(g, va); continue; } pte_store_release(&l2[l2_idx], make_block_desc(cur_gpa, perms)); + guest_pte_present_set(g, va); if (!bcast) { if (va < changed_lo) changed_lo = va; @@ -1361,8 +1428,10 @@ typedef struct { * * Single-entry translation cache: avoids 3-4 pointer chases through the page * table on repeated accesses to the same 2MiB block (or 4KiB page if L3-split). - * Validated by an atomic generation counter in guest_t that is bumped on every - * page table modification. + * Validated by two generation counters: guest_t.pt_gen, which host page-table + * writers bump under mmap_lock, and the EL1 PT epoch in shim_data, which the + * EL1 munmap fast path bumps after clearing descriptors with no host round + * trip. Either one moving retires the entry. */ static _Thread_local struct { const guest_t *owner; /* Which guest_t this entry belongs to */ @@ -1371,8 +1440,16 @@ static _Thread_local struct { uint64_t size; /* 2MiB or 4KiB (0 = invalid) */ int perms; /* Cached permissions */ uint64_t gen; /* guest_t.pt_gen at population time */ + uint64_t el1_epoch; /* SHIM_MMAP_PT_EPOCH_OFF word at population time */ } gva_tlb; +static const _Atomic uint64_t *guest_el1_pt_epoch(const guest_t *g) +{ + return (const _Atomic uint64_t *) ((const uint8_t *) g->host_base + + g->shim_data_base + + SHIM_MMAP_PT_EPOCH_OFF); +} + static void guest_tlb_flush(void) { gva_tlb.size = 0; @@ -1383,10 +1460,16 @@ static int gva_translate_perm(const guest_t *g, int required_perms, gva_translation_t *out) { - /* Fast path: check per-thread TLB cache */ + /* Fast path: check per-thread TLB cache. Both counters are read before the + * walk below, so a clear that lands during the walk leaves the entry + * recording a value the counter has already moved past. + */ uint64_t gen = atomic_load_explicit(&g->pt_gen, memory_order_acquire); + uint64_t el1_epoch = + atomic_load_explicit(guest_el1_pt_epoch(g), memory_order_acquire); if (gva_tlb.size && gva_tlb.owner == g && gva_tlb.gen == gen && - gva >= gva_tlb.base_gva && gva - gva_tlb.base_gva < gva_tlb.size && + gva_tlb.el1_epoch == el1_epoch && gva >= gva_tlb.base_gva && + gva - gva_tlb.base_gva < gva_tlb.size && (required_perms & gva_tlb.perms) == required_perms) { out->gpa = gva_tlb.base_gpa + (gva - gva_tlb.base_gva); out->chunk = (gva_tlb.base_gva + gva_tlb.size) - gva; @@ -1473,6 +1556,7 @@ static int gva_translate_perm(const guest_t *g, gva_tlb.size = PAGE_SIZE; gva_tlb.perms = perms; gva_tlb.gen = gen; + gva_tlb.el1_epoch = el1_epoch; return 0; } @@ -1506,6 +1590,7 @@ static int gva_translate_perm(const guest_t *g, gva_tlb.size = BLOCK_2MIB; gva_tlb.perms = perms; gva_tlb.gen = gen; + gva_tlb.el1_epoch = el1_epoch; return 0; } @@ -1586,11 +1671,11 @@ static uint64_t gva_contiguous_avail(const guest_t *g, * (MEM_PERM_R/W/X bitmask). The walk continues across adjacent L2/L3 entries * until a mapping, permission, or physical-contiguity break is found. */ -static void *gva_resolve_perm(const guest_t *g, - uint64_t gva, - uint64_t *avail, - int required_perms, - uint64_t avail_limit) +static void *gva_resolve_perm_walk(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit) { /* Always walk page tables to enforce permissions. The guest slab is * identity-mapped (GVA == GPA == offset), but L2 block descriptors carry @@ -1651,14 +1736,122 @@ static void *gva_resolve_perm(const guest_t *g, return NULL; } +/* Host-access fault-in for lazy (deferred-PTE) regions. + * + * A syscall may target guest memory the guest itself has never touched: a fresh + * calloc()-style arena handed straight to read(2), a futex word inside an + * untouched mapping, an iovec into a new heap chunk. The guest-fault path + * (guest_materialize_lazy via the EL1 shim) never runs for those, so the + * page-table walk in gva_resolve_perm_walk fails even though the access is + * legal. Materialize the touched blocks here, then let the caller re-walk. + * + * Locking: the fault-in hook takes mmap_lock unless the caller already owns it, + * in which case it fails. Callers holding locks that nest below mmap_lock must + * pre-fault outside those locks and use nofault access inside them. + * + * TLBI: guest_materialize_lazy accumulates TLBI requests in the calling + * thread's per-vCPU slot. On a vCPU thread the syscall epilogue emits them. On + * non-vCPU threads the request is lost, which is self-healing: a vCPU that + * still holds a stale negative TLB entry re-faults, and the already-valid + * early-return in guest_materialize_lazy re-issues a page TLBI for it without + * re-zeroing. + * + * Returns 0 if at least one block in [gva, gva+len) is now materialized (or + * already was), -1 if the range intersects no materializable lazy region. + */ +int guest_lazy_faultin_locked(const guest_t *cg, uint64_t gva, uint64_t len) +{ + /* The lazy machinery mutates page tables; the const on the resolve API + * reflects the pure-walk fast path, not this slow path. + */ + guest_t *g = (guest_t *) (uintptr_t) cg; + + if (gva >= g->guest_size) + return -1; /* High-VA / non-identity ranges are never lazy. */ + if (len == 0) + len = 1; + uint64_t end = (len > g->guest_size - gva) ? g->guest_size : gva + len; + + int rc = -1; + for (uint64_t b = gva & ~(uint64_t) (BLOCK_2MIB - 1); b < end; + b += BLOCK_2MIB) { + uint64_t probe = (b > gva) ? b : gva; + if (guest_materialize_lazy(g, probe) == 0) + rc = 0; + } + return rc; +} + +static int gva_lazy_faultin(const guest_t *cg, + uint64_t gva, + uint64_t len, + int required_perms) +{ + (void) required_perms; /* Region prot gates the retry walk, not this. */ + + if (mmap_lock_held_by_current_thread()) + return -1; + + int rc; + mmap_lock_acquire((guest_t *) (uintptr_t) cg); + rc = guest_lazy_faultin_locked(cg, gva, len); + mmap_lock_release(); + return rc; +} + +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len) +{ + uint64_t avail = 0; + uint64_t want = len ? len : 1; + if (gva_resolve_perm_walk(g, gva, &avail, MEM_PERM_R, want) && + avail >= want) + return 0; + return gva_lazy_faultin(g, gva, len, MEM_PERM_R); +} + +static void *gva_resolve_perm(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit, + bool allow_faultin) +{ + void *ptr = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (!allow_faultin) + return ptr; + + /* Window the fault-in to what the caller actually needs. Length-less + * resolves (guest_ptr / guest_ptr_avail) materialize a single block; their + * callers iterate and re-enter here per chunk. + */ + uint64_t want = (avail_limit == UINT64_MAX) ? 1 : avail_limit; + if (want == 0) + return ptr; + if (!ptr) { + if (gva_lazy_faultin(g, gva, want, required_perms) < 0) + return NULL; + return gva_resolve_perm_walk(g, gva, avail, required_perms, + avail_limit); + } + if (avail && *avail < want && gva <= UINT64_MAX - *avail && + gva_lazy_faultin(g, gva + *avail, want - *avail, required_perms) == 0) { + void *again = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (again) + ptr = again; + } + return ptr; +} + void *guest_ptr(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX, true); } void *guest_ptr_w(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX, true); } void *guest_ptr_avail(const guest_t *g, @@ -1666,7 +1859,19 @@ void *guest_ptr_avail(const guest_t *g, uint64_t *avail, int required_perms) { - return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX); + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, true); +} + +/* Pure page-table walk without lazy fault-in. For callers that already hold + * mmap_lock or must avoid acquiring it (e.g. the stale-TLB re-walk in the EL0 + * fault handler, which runs after guest_materialize_lazy has been consulted). + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms) +{ + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, false); } void *guest_ptr_bound(const guest_t *g, @@ -1675,7 +1880,7 @@ void *guest_ptr_bound(const guest_t *g, int required_perms, uint64_t len_limit) { - return gva_resolve_perm(g, gva, avail, required_perms, len_limit); + return gva_resolve_perm(g, gva, avail, required_perms, len_limit, true); } static inline int guest_copy(const guest_t *g, @@ -1683,7 +1888,8 @@ static inline int guest_copy(const guest_t *g, void *dst, const void *src, size_t len, - int required_perms) + int required_perms, + bool allow_faultin) { if (len == 0) return 0; @@ -1697,7 +1903,7 @@ static inline int guest_copy(const guest_t *g, while (copied < len) { uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, required_perms, - (uint64_t) (len - copied)); + (uint64_t) (len - copied), allow_faultin); if (!ptr) return -1; size_t chunk = len - copied; @@ -1717,7 +1923,12 @@ static inline int guest_copy(const guest_t *g, int guest_read(const guest_t *g, uint64_t gva, void *dst, size_t len) { - return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R); + return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R, true); +} + +int guest_read_nofault(const guest_t *g, uint64_t gva, void *dst, size_t len) +{ + return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R, false); } int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) @@ -1733,19 +1944,25 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len) { - return guest_copy(g, gva, NULL, src, len, MEM_PERM_W); + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, true); } -size_t guest_write_partial(guest_t *g, - uint64_t gva, - const void *src, - size_t len) +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len) +{ + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, false); +} + +static size_t guest_write_partial_impl(guest_t *g, + uint64_t gva, + const void *src, + size_t len, + bool allow_faultin) { size_t done = 0; while (done < len) { uint64_t avail; void *dst = gva_resolve_perm(g, gva + done, &avail, MEM_PERM_W, - (uint64_t) (len - done)); + (uint64_t) (len - done), allow_faultin); if (!dst) return done; @@ -1761,6 +1978,22 @@ size_t guest_write_partial(guest_t *g, return done; } +size_t guest_write_partial(guest_t *g, + uint64_t gva, + const void *src, + size_t len) +{ + return guest_write_partial_impl(g, gva, src, len, true); +} + +size_t guest_write_partial_nofault(guest_t *g, + uint64_t gva, + const void *src, + size_t len) +{ + return guest_write_partial_impl(g, gva, src, len, false); +} + int guest_write_small(guest_t *g, uint64_t gva, const void *src, size_t len) { uint64_t avail = 0; @@ -1789,7 +2022,7 @@ int guest_read_str(const guest_t *g, uint64_t gva, char *dst, size_t max) break; uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, MEM_PERM_R, - (uint64_t) (limit - copied)); + (uint64_t) (limit - copied), true); if (!ptr) break; @@ -1879,6 +2112,7 @@ void guest_reset(guest_t *g) if (gpa > g->guest_size || len > g->guest_size - gpa) continue; /* backing lies outside the primary slab */ memset((uint8_t *) g->host_base + gpa, 0, len); + guest_dirty_clear_zeroed_range(g, gpa, gpa + len); } /* Zero page table pool (not tracked in region array) */ @@ -1915,6 +2149,8 @@ void guest_reset(guest_t *g) g->mmap_rw_gap_hint = 0; g->mmap_rx_gap_hint = 0; g->ttbr0 = 0; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); tlbi_request_clear(); g->elf_load_min = ELF_DEFAULT_BASE; @@ -2902,6 +3138,9 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) { uint64_t base = g->ipa_base; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + /* Allocate L0 table */ uint64_t l0_gpa = pt_alloc_page(g); if (!l0_gpa) @@ -2998,6 +3237,8 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) * to the primary-buffer GPA where the bytes actually are. */ l2[l2_idx] = make_block_desc(output_ipa, block_perms); + if (lookup_addr >= base) + guest_pte_present_set(g, lookup_addr - base); } } @@ -3011,6 +3252,11 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) if (!finalize_block_perms(g, regions, n)) return 0; + for (int r = 0; r < n; r++) { + if (regions[r].perms & MEM_PERM_W) + guest_dirty_mark_range(g, regions[r].gpa_start, regions[r].gpa_end); + } + guest_pt_gen_bump(g); return ttbr0; } @@ -3129,9 +3375,12 @@ int guest_extend_page_tables(guest_t *g, * an explicit PT_VALID test so the intent survives a future * descriptor-bit renumbering. */ - if (l2[l2_idx] & PT_VALID) + if (l2[l2_idx] & PT_VALID) { + guest_pte_present_set(g, addr); continue; + } pte_store_release(&l2[l2_idx], make_block_desc(ipa, perms)); + guest_pte_present_set(g, addr); if (!bcast) { if (addr < changed_lo) changed_lo = addr; @@ -3147,10 +3396,129 @@ int guest_extend_page_tables(guest_t *g, return 0; } +static uint64_t guest_va_next_page_table_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || !g->ttbr0) + return end; + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + if (!l0) + return end; + + va &= ~(uint64_t) (BLOCK_2MIB - 1); + while (va < end) { + uint64_t ipa = base + va; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return end; + if (!(l0[l0_idx] & PT_VALID)) { + uint64_t next_ipa = (uint64_t) (l0_idx + 1) * 512ULL * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; /* wrap guard */ + va = next_ipa - base; + continue; + } + uint64_t *l1 = pt_at(g, (l0[l0_idx] & 0xFFFFFFFFF000ULL) - base); + if (!l1) + return end; + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + if (!(l1[l1_idx] & PT_VALID)) { + uint64_t next_ipa = (ipa / BLOCK_1GIB + 1) * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; + va = next_ipa - base; + continue; + } + return va; + } + return end; +} + +static uint64_t guest_va_next_indexed_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (va >= end || va >= GUEST_PTE_PRESENT_LIMIT) + return end; + if (end > GUEST_PTE_PRESENT_LIMIT) + end = GUEST_PTE_PRESENT_LIMIT; + + uint64_t first_block = va / BLOCK_2MIB; + uint64_t last_block = (end - 1) / BLOCK_2MIB; + uint64_t first_word = first_block >> 6; + uint64_t last_word = last_block >> 6; + uint64_t bits = + g->pte_present_blocks[first_word] & (~0ULL << (first_block & 63)); + if (first_word == last_word && (last_block & 63) != 63) + bits &= (1ULL << ((last_block & 63) + 1)) - 1; + if (bits) + return (first_word * 64 + (uint64_t) __builtin_ctzll(bits)) * + BLOCK_2MIB; + + uint64_t word = first_word + 1; + while (word <= last_word) { + uint64_t summary_word = word >> 6; + uint64_t summary_last = last_word >> 6; + uint64_t summary = + g->pte_present_summary[summary_word] & (~0ULL << (word & 63)); + if (summary_word == summary_last && (last_word & 63) != 63) + summary &= (1ULL << ((last_word & 63) + 1)) - 1; + if (summary) { + uint64_t present_word = + summary_word * 64 + (uint64_t) __builtin_ctzll(summary); + uint64_t present = g->pte_present_blocks[present_word]; + if (present_word == last_word && (last_block & 63) != 63) + present &= (1ULL << ((last_block & 63) + 1)) - 1; + if (present) + return (present_word * 64 + + (uint64_t) __builtin_ctzll(present)) * + BLOCK_2MIB; + } + if (summary_word == summary_last) + break; + word = (summary_word + 1) * 64; + } + return end; +} + +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || va >= end) + return end; + if (va & (BLOCK_2MIB - 1)) { + if (va > UINT64_MAX - (BLOCK_2MIB - 1)) + return end; + va = ALIGN_2MIB_UP(va); + if (va >= end) + return end; + } + + if (va < GUEST_PTE_PRESENT_LIMIT) { + uint64_t low_end = + end < GUEST_PTE_PRESENT_LIMIT ? end : GUEST_PTE_PRESENT_LIMIT; + uint64_t next = guest_va_next_indexed_block(g, va, low_end); + if (next < low_end || end <= GUEST_PTE_PRESENT_LIMIT) + return next; + va = GUEST_PTE_PRESENT_LIMIT; + } + + /* Non-identity/high-VA mappings live outside the compact low-VA index. + * Preserve the page-table walker for those uncommon ranges. + */ + return guest_va_next_page_table_block(g, va, end); +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) return false; + if (va < GUEST_PTE_PRESENT_LIMIT) + return guest_pte_present_test(g, va); uint64_t base = g->ipa_base; uint64_t *l0 = pt_at(g, g->ttbr0 - base); @@ -3179,6 +3547,71 @@ bool guest_va_block_mapped(const guest_t *g, uint64_t va) return (l2[l2_idx] & PT_VALID) != 0; } +void guest_rebuild_pte_present(guest_t *g) +{ + if (!g) + return; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + if (!g->ttbr0) + return; + + uint64_t limit = g->guest_size; + if (limit > GUEST_PTE_PRESENT_LIMIT) + limit = GUEST_PTE_PRESENT_LIMIT; + for (uint64_t va = 0; va < limit; va += BLOCK_2MIB) { + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + continue; + if ((*l2_entry & 3) == 1) { + guest_pte_present_set(g, va); + continue; + } + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (!l3) + continue; + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + guest_pte_present_set(g, va); + break; + } + } + } +} + +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t block = ALIGN_2MIB_DOWN(start); + while (block < end) { + bool present = false; + uint64_t *l2_entry = find_l2_entry(g, block); + if (l2_entry && (*l2_entry & PT_VALID)) { + if ((*l2_entry & 3) == 1) { + present = true; + } else { + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (l3) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (pte_load_acquire(&l3[i]) & PT_VALID) { + present = true; + break; + } + } + } + } + } + if (present) + guest_pte_present_set(g, block); + else + guest_pte_present_clear(g, block); + block += BLOCK_2MIB; + } +} + /* L3 page table splitting. */ /* L3 page descriptor: bits[1:0]=11 = valid page at level 3. This is distinct @@ -3335,6 +3768,7 @@ int guest_split_block(guest_t *g, uint64_t block_gpa) int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) { uint64_t base = g->ipa_base; + bool any_changed = false; /* Page-align the range. The ALIGN_UP step on end could wrap to 0 for inputs * within PAGE_SIZE-1 of UINT64_MAX, silently turning the invalidation into @@ -3349,10 +3783,21 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) return 0; for (uint64_t addr = start; addr < end;) { + uint64_t indexed_block = ALIGN_2MIB_DOWN(addr); + if (indexed_block < GUEST_PTE_PRESENT_LIMIT && + !guest_pte_present_test(g, indexed_block)) { + addr = + guest_va_next_present_block(g, indexed_block + BLOCK_2MIB, end); + continue; + } uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 entry (already unmapped); skip this 2MiB block */ - addr = ALIGN_2MIB_UP(addr + 1); + /* No L2 table: nothing to invalidate in this block. The low-VA + * occupancy index jumps directly to the next block containing a + * valid PTE; high VA retains the page-table hierarchy fallback. + */ + guest_pte_present_clear(g, indexed_block); + addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3361,6 +3806,7 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) /* Not mapped at all: skip */ if (!(*l2_entry & 1)) { + guest_pte_present_clear(g, block_start); addr = block_end; continue; } @@ -3374,6 +3820,8 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) * broadcast. */ pte_store_release(l2_entry, 0); + guest_pte_present_clear(g, block_start); + any_changed = true; tlbi_request_range(base + block_start, base + block_end); addr = block_end; continue; @@ -3399,12 +3847,14 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) uint64_t page_end = (end < block_end) ? end : block_end; uint64_t changed_lo = UINT64_MAX, changed_hi = 0; bool bcast = tlbi_request_is_broadcast(); + bool block_changed = false; for (uint64_t pa = page_start; pa < page_end; pa += PAGE_SIZE) { unsigned l3_idx = (unsigned) (((base + pa) % BLOCK_2MIB) / PAGE_SIZE); if (l3[l3_idx] != 0) { pte_store_release(&l3[l3_idx], 0); /* Invalid descriptor */ + block_changed = true; if (!bcast) { if (pa < changed_lo) changed_lo = pa; @@ -3416,10 +3866,27 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + if (block_changed) + any_changed = true; + + bool block_present = false; + if (page_start > block_start || page_end < block_end) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + block_present = true; + break; + } + } + } + if (block_present) + guest_pte_present_set(g, block_start); + else + guest_pte_present_clear(g, block_start); addr = page_end; } - guest_pt_gen_bump(g); + if (any_changed) + guest_pt_gen_bump(g); return 0; } @@ -3495,6 +3962,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) pte_store_release(l2_entry, make_block_desc(ipa, perms)); tlbi_request_range(base + block_start, base + block_end); } + guest_pte_present_set(g, block_start); addr = block_end; continue; } @@ -3569,9 +4037,15 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + guest_pte_present_set(g, block_start); addr = page_end; } + /* Writable PTEs allow guest stores without another host fault. Keep the + * block dirty even when materialization has just zeroed its backing. + */ + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, start, end); guest_pt_gen_bump(g); return 0; } @@ -3650,19 +4124,136 @@ int guest_install_va_pages(guest_t *g, changed_hi = v + PAGE_SIZE; } } + guest_pte_present_set(g, v); } if (!bcast && changed_hi > changed_lo) tlbi_request_range(changed_lo, changed_hi); + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, gpa, gpa + length); guest_pt_gen_bump(g); return 0; } -/* Lazy page materialization for MAP_NORESERVE. */ +/* Lazy page materialization for deferred-PTE (private anonymous / + * MAP_NORESERVE) regions. + */ -int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start) +{ + if (!g || block_start >= g->guest_size) + return true; + uint64_t block = block_start / BLOCK_2MIB; + return (g->dirty_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = start / BLOCK_2MIB; + uint64_t last = (end - 1) / BLOCK_2MIB; + for (uint64_t block = first; block <= last; block++) + g->dirty_blocks[block >> 6] |= 1ULL << (block & 63); +} + +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = ALIGN_2MIB_UP(start); + uint64_t last = ALIGN_2MIB_DOWN(end); + for (uint64_t addr = first; addr < last; addr += BLOCK_2MIB) { + uint64_t block = addr / BLOCK_2MIB; + g->dirty_blocks[block >> 6] &= ~(1ULL << (block & 63)); + } +} + +static bool materialize_claim_overlaps(const guest_materialize_claim_t *claim, + uint64_t start, + uint64_t end) +{ + return claim->active && start < claim->end && end > claim->start; +} + +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (;;) { + bool overlap = false; + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], start, + end)) { + overlap = true; + break; + } + } + if (!overlap) + return; + mmap_lock_cond_wait(g, &g->materialize_cond); + } +} + +void guest_materialize_wait_all_locked(guest_t *g) +{ + guest_materialize_wait_range_locked(g, 0, UINT64_MAX); +} + +static int materialize_claim_alloc_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + guest_materialize_wait_range_locked(g, start, end); + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + guest_materialize_claim_t *claim = &g->materialize_claims[i]; + if (!claim->active) { + claim->start = start; + claim->end = end; + claim->active = true; + return i; + } + } + return -1; +} + +static void materialize_claim_release_locked(guest_t *g, int slot) +{ + if (slot < 0) + return; + g->materialize_claims[slot].active = false; + pthread_cond_broadcast(&g->materialize_cond); +} + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Callers + * must hold mmap_lock; used to detect blocks a concurrent fault already + * materialized. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va) +{ + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + return false; + if ((*l2_entry & 3) == 1) + return true; /* 2MiB block descriptor */ + uint64_t *l3 = pt_at(g, (*l2_entry & 0xFFFFFFFFF000ULL) - g->ipa_base); + if (!l3) + return false; + unsigned l3_idx = + (unsigned) (((g->ipa_base + va) % BLOCK_2MIB) / PAGE_SIZE); + return (l3[l3_idx] & PT_VALID) != 0; +} + +static int guest_materialize_lazy_one(guest_t *g, uint64_t fault_offset) { - /* Find the noreserve region containing this offset */ +retry:; + /* Find the lazy region containing this offset */ const guest_region_t *region = NULL; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start <= fault_offset && @@ -3673,7 +4264,36 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) } if (!region) - return -1; /* Not a noreserve region */ + return -1; /* Not a lazy region */ + + /* PROT_NONE is a reservation, not a mapping: a fault inside it is a genuine + * SIGSEGV, never a materialization request. Without this check a + * PROT_NONE|MAP_NORESERVE region would be silently granted read permission + * by the perms fallback below. + */ + if (region->prot == LINUX_PROT_NONE) + return -1; + + uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); + uint64_t block_end = block_start + BLOCK_2MIB; + if (block_end > g->guest_size) + block_end = g->guest_size; + + /* Already materialized: another thread (concurrent guest fault or a + * host-side fault-in on a syscall path) completed this block while this + * vCPU was queued on mmap_lock, and the guest may have written real data + * through the new PTEs since. Running the memset below again would wipe + * those writes. The fault that got us here is then either a stale negative + * TLB entry or an in-flight retry. Invalidate the whole materialization + * block so this path follows the same one-RVAE-per-block contract as a + * newly installed block. + */ + if (guest_va_pte_valid(g, fault_offset)) { + mmap_fastpath_note_materialized_locked(g, block_start, block_end); + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + return 0; + } /* Materialize one 2MiB block containing the fault address. This is the * smallest granule that guest_extend_page_tables works with. For the common @@ -3681,12 +4301,18 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) * trade-off: it avoids over-committing the large reservation while keeping * the fault rate manageable. */ - uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); - uint64_t block_end = block_start + BLOCK_2MIB; - /* Clamp to guest size */ - if (block_end > g->guest_size) - block_end = g->guest_size; + /* A sibling may be zeroing another window in this block without the lock. + * Wait before inspecting regions/PTEs, then restart because a mutator that + * was itself waiting may have changed the region layout first. + */ + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], block_start, + block_end)) { + guest_materialize_wait_range_locked(g, block_start, block_end); + goto retry; + } + } uint64_t materialize_start = (block_start > region->start) ? block_start : region->start; @@ -3707,18 +4333,70 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) if (perms == 0) perms = MEM_PERM_R; /* At minimum readable */ + /* Zero the window BEFORE any PTE becomes valid. The moment a descriptor is + * published, sibling vCPUs with no stale TLB entry can write through it + * without ever faulting; zeroing afterwards (the historical order) would + * wipe such a write. The slab is host memory, so zeroing needs no PTEs, and + * every writer that could touch the window first goes through mmap_lock + * (guest faults and host-side fault-in alike), so nothing can write between + * this memset and the descriptor stores below. Skip pages that are already + * valid: they belong to a previously materialized neighbor in the same + * block and may hold live data. + */ + int claim_slot = -1; + bool dirty = guest_block_may_be_dirty(g, block_start); + if (dirty) { + uint64_t zero_pages[8] = {0}; + bool any_valid = false; + for (uint64_t pg = materialize_start; pg < materialize_end; + pg += PAGE_SIZE) { + unsigned page = (unsigned) ((pg - block_start) / PAGE_SIZE); + if (guest_va_pte_valid(g, pg)) + any_valid = true; + else + zero_pages[page >> 6] |= 1ULL << (page & 63); + } + + claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); + if (claim_slot >= 0) + mmap_lock_drop_keep_gate(); + for (unsigned page = 0; page < 512;) { + if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { + page++; + continue; + } + unsigned first = page; + do { + page++; + } while (page < 512 && + (zero_pages[page >> 6] & (1ULL << (page & 63)))); + memset((uint8_t *) g->host_base + block_start + + (uint64_t) first * PAGE_SIZE, + 0, (uint64_t) (page - first) * PAGE_SIZE); + } + if (claim_slot >= 0) + mmap_lock_reacquire_with_gate(g); + if (!(perms & MEM_PERM_W) && !any_valid && + materialize_start == block_start && materialize_end == block_end) + guest_dirty_clear_zeroed_range(g, block_start, block_end); + } + /* Create page table entries. guest_extend_page_tables creates L2 block * descriptors but skips existing table descriptors (L2->L3 splits). * guest_update_perms handles the L3 case: if guest_invalidate_ptes * previously split the block and invalidated the L3 entries, update_perms * recreates them with correct perms. */ - if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) + if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (partial_block) { - if (guest_split_block(g, block_start) < 0) + if (guest_split_block(g, block_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } /* If this block had no page-table entry before the lazy fault, * guest_extend_page_tables() necessarily created a full 2MiB block. @@ -3728,25 +4406,103 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) */ if (!had_mapping) { if (block_start < materialize_start && - guest_invalidate_ptes(g, block_start, materialize_start) < 0) + guest_invalidate_ptes(g, block_start, materialize_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (materialize_end < block_end && - guest_invalidate_ptes(g, materialize_end, block_end) < 0) + guest_invalidate_ptes(g, materialize_end, block_end) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } } } guest_update_perms(g, materialize_start, materialize_end, perms); - /* Zero the materialized memory. Only zero within the region boundaries to - * avoid clobbering adjacent data. - */ - if (materialize_end > materialize_start) - memset((uint8_t *) g->host_base + materialize_start, 0, - materialize_end - materialize_start); - - /* The page-table helpers above already requested the matching TLBI; no - * additional flush is needed here. + /* One 2MiB materialization gets one block-sized RVAE1IS. This is cheaper + * than per-page invalidation and covers every negative entry that may have + * been cached for the block while its descriptors were invalid. */ + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + g->materialize_stats[dirty ? GUEST_MATERIALIZE_DIRTY_MEMSET + : GUEST_MATERIALIZE_CLEAN_SKIP]++; + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += + materialize_end - materialize_start; + mmap_fastpath_note_materialized_locked(g, materialize_start, + materialize_end); + materialize_claim_release_locked(g, claim_slot); return 0; } + +int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +{ + return guest_materialize_lazy_one(g, fault_offset); +} + +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset) +{ + typedef struct { + guest_t *guest; + uint64_t next_block; + unsigned streak; + } fault_around_state_t; + static _Thread_local fault_around_state_t state; + + uint64_t block = ALIGN_2MIB_DOWN(fault_offset); + if (state.guest == g && block == state.next_block) { + if (state.streak < 4) + state.streak++; + } else { + state.guest = g; + state.streak = 0; + } + + const guest_region_t *region = guest_region_find(g, fault_offset); + if (!region || !region->noreserve || region->prot == LINUX_PROT_NONE) + return -1; + + unsigned blocks = 1U << state.streak; + if (blocks > 16) + blocks = 16; + uint64_t region_last = ALIGN_2MIB_UP(region->end); + if (region_last > g->guest_size) + region_last = g->guest_size; + + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + if (guest_block_may_be_dirty(g, ahead) && blocks > 4) { + blocks = 4; + break; + } + } + + int result = -1; + uint64_t last = block; + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + + /* The current block must probe the actual FAR. A fast-path mmap can + * extend an already materialized region within the same 2MiB block; + * probing the region/block start would see an older valid page and + * return without installing the newly faulted page. Ahead blocks have + * no FAR, so use their first address inside the region. + */ + uint64_t probe = (i == 0) + ? fault_offset + : (ahead < region->start ? region->start : ahead); + int rc = guest_materialize_lazy_one(g, probe); + if (i == 0) + result = rc; + if (rc < 0) + break; + last = ahead; + } + if (result == 0) + state.next_block = last + BLOCK_2MIB; + return result; +} diff --git a/src/core/guest.h b/src/core/guest.h index f265fabd..bc0c147c 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -264,7 +265,11 @@ typedef struct { uint64_t offset; /* File offset (for /proc/self/maps display) */ int backing_fd; /* Duplicated host fd for file-backed mappings, or -1 */ bool shared; /* MAP_SHARED (writes should propagate) */ - bool noreserve; /* MAP_NORESERVE: PTEs deferred until fault */ + bool noreserve; /* Lazy region: PTEs and zeroing deferred until first + * touch. Set for MAP_NORESERVE and for all private + * anonymous mappings (sys_mmap folds the latter into + * the tracked MAP_NORESERVE bit; not guest visible). + */ bool backing_ro; /* MAP_SHARED region whose backing_fd was opened * without write access, so its Linux max_prot is * capped to PROT_READ. sys_mprotect must reject any @@ -301,6 +306,8 @@ typedef struct { * TLBI_BROADCAST -> X8 = 1 (TLBI VMALLE1IS, broadest) * TLBI_RANGE -> X8 = 3, X9 = start VA, X10 = page count * (TLBI VAE1IS loop preserves unrelated TLB entries) + * TLBI_RANGE_LARGE -> X8 = 4, X9 = encoded range operand + * (single TLBI RVAE1IS) * X8 = 2 is reserved for the execve drop-frame marker the shim handles * separately; it is never produced by the accumulator. */ @@ -323,14 +330,12 @@ typedef enum { */ #define TLBI_SELECTIVE_MAX_PAGES 16 -/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. With SCALE=0 the - * RVAE1IS operand encoding covers (NUM+1)*2 pages with NUM in [0..31], so a - * single instruction reaches 64 pages == 256 KiB. Beyond that the host would - * need SCALE=1 (NUM*64 step), which over-invalidates for the typical - * dynamic-linker RELRO / glibc-bring-up storm sizes seen in practice; stay at - * SCALE=0 for now and broadcast above 64 pages. +/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. SCALE=0 covers up to + * 64 pages, SCALE=1 up to 2048 pages, and SCALE=2 much larger ranges. 32768 + * pages (128 MiB) covers the lazy fault-around window with one instruction + * while still fitting tlbi_request_t.pages in uint16_t. */ -#define TLBI_RVAE_MAX_PAGES 64 +#define TLBI_RVAE_MAX_PAGES 32768 /* TLBI RVAE1IS operand bit-field constants. Per ARM ARM DDI 0487J.a D8.7.6 the * operand layout is: @@ -346,28 +351,37 @@ typedef enum { */ #define RVAE_OPERAND_BADDR_MASK ((1ULL << 37) - 1) #define RVAE_OPERAND_NUM_SHIFT 39 +#define RVAE_OPERAND_SCALE_SHIFT 44 #define RVAE_OPERAND_TG_4KB (1ULL << 46) /* Pure encoder: build the TLBI RVAE1IS Xt operand from a 4 KiB-aligned VA and a - * page count in the SCALE=0 range (1..TLBI_RVAE_MAX_PAGES). Lives in the header - * as static inline so tlbi_request_emit_to_vcpu and any future caller - * (host-side unit tests included) compile to the same expression. NUM = - * ceil(pages / 2) - 1 over-invalidates odd page counts by exactly one page, - * which is a perf-only side effect (the extra invalidation evicts a neighbour - * TLB entry that the guest's next access reloads). pages < 2 is clamped to 2 - * because SCALE=0 NUM=0 means 2 pages -- the encoder cannot represent a single - * page through RVAE1IS; single-page callers go through the per-page VAE1IS path - * instead, but the clamp keeps the encoder total in any pathological input. + * page count in the supported SCALE=0..2 range. Lives in the header so the emit + * path and host unit tests use the same expression. Each NUM step covers + * 2^(5*SCALE+1) pages; the normalizer aligns and widens callers to that unit. + * pages < 2 is clamped to 2 because SCALE=0 NUM=0 is the smallest encodable + * range. Single-page callers normally use VAE1IS instead. */ +static inline uint64_t tlbi_rvae_unit_pages(uint64_t pages) +{ + if (pages <= 64) + return 2; /* SCALE=0 */ + if (pages <= 2048) + return 64; /* SCALE=1 */ + return 2048; /* SCALE=2 */ +} + static inline uint64_t tlbi_rvae1is_operand(uint64_t start_va, uint16_t pages) { if (pages < 2) pages = 2; + uint64_t scale = pages <= 64 ? 0 : (pages <= 2048 ? 1 : 2); + uint64_t unit_pages = 1ULL << (5 * scale + 1); uint64_t baddr = (start_va >> 12) & RVAE_OPERAND_BADDR_MASK; - uint64_t num = ((pages + 1) / 2) - 1; + uint64_t num = ((pages + unit_pages - 1) / unit_pages) - 1; if (num > 31) num = 31; - return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | RVAE_OPERAND_TG_4KB; + return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | + (scale << RVAE_OPERAND_SCALE_SHIFT) | RVAE_OPERAND_TG_4KB; } /* Runtime feature flag: TRUE when the host PE implements FEAT_TLBIRANGE @@ -383,8 +397,8 @@ typedef struct { * after the TLBI sequence. 0 = data-only * change, skip the I-cache invalidation. */ - uint16_t pages; /* Page count when kind == TLBI_RANGE (1..MAX) */ - uint64_t start; /* Page-aligned VA when kind == TLBI_RANGE */ + uint16_t pages; /* Page count for either range kind (1..MAX) */ + uint64_t start; /* Page-aligned VA for either range kind */ } tlbi_request_t; /* Layout contract: 16 bytes (1+1+2+4 padding+8). Documents the padding and pins @@ -437,6 +451,44 @@ typedef struct { uint64_t next; /* Bump offset; (next + BLOCK_2MIB) > size means full */ } guest_overflow_t; +/* One conservative "may contain nonzero bytes" bit per 2 MiB primary-slab + * block. The largest supported slab is 1 TiB, so this costs 64 KiB per guest. + * All access is serialized by mmap_lock. + */ +#define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) +#define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) + +/* Host-side occupancy index for low-VA TTBR0 mappings. One bit per 2 MiB block + * records whether that block contains at least one valid L2/L3 PTE; a + * second-level bitmap records which occupancy words are non-zero. This lets + * huge lazy mmap/munmap ranges skip untouched address space without walking one + * page-table slot per GiB. The index is separate from dirty_blocks: a read-only + * mapping can have valid PTEs without dirty backing bytes. + */ +#define GUEST_PTE_PRESENT_BLOCKS_MAX GUEST_DIRTY_BLOCKS_MAX +#define GUEST_PTE_PRESENT_WORDS (GUEST_PTE_PRESENT_BLOCKS_MAX / 64) +#define GUEST_PTE_PRESENT_SUMMARY_WORDS (GUEST_PTE_PRESENT_WORDS / 64) +#define GUEST_PTE_PRESENT_LIMIT (GUEST_PTE_PRESENT_BLOCKS_MAX * BLOCK_2MIB) + +enum { + GUEST_MATERIALIZE_CLEAN_SKIP = 0, + GUEST_MATERIALIZE_DIRTY_MEMSET, + GUEST_MATERIALIZE_ALREADY_VALID, + GUEST_MATERIALIZE_WINDOW_BYTES, + GUEST_MATERIALIZE_STATS_N, +}; + +/* Dirty-block zeroing claims. A claim makes its block's invalid PTE window + * stable while the expensive host memset runs without mmap_lock. Waiters use + * one guest-wide condition variable; the fixed table bounds host allocation and + * is ample for the vCPU limit. + */ +#define GUEST_MATERIALIZE_CLAIMS 64 +typedef struct { + uint64_t start, end; + bool active; +} guest_materialize_claim_t; + /* Guest state. */ typedef struct { void *host_base; /* Host pointer to allocated guest memory */ @@ -558,12 +610,20 @@ typedef struct { hvf_segment_t segments[GUEST_MAX_HVF_SEGMENTS]; int n_segments; - /* Page table generation counter: incremented on every PT modification. Used - * by the per-thread GVA TLB cache to detect stale entries. 64-bit to avoid - * wrap-around stale hits over long-running sessions. + /* Page table generation counter: incremented on every host PT modification. + * The per-thread GVA TLB cache checks it together with the EL1 PT epoch at + * SHIM_MMAP_PT_EPOCH_OFF, which EL1 fast munmap bumps in its place. 64-bit + * to avoid wrap-around stale hits over long-running sessions. */ _Atomic uint64_t pt_gen; + uint64_t pte_present_blocks[GUEST_PTE_PRESENT_WORDS]; + uint64_t pte_present_summary[GUEST_PTE_PRESENT_SUMMARY_WORDS]; + uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; + uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; + guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; + pthread_cond_t materialize_cond; + /* Optional HVC 6 embedder extension hook. * * Native AArch64 guests reach this through HVC 6. When the build enables @@ -586,8 +646,8 @@ typedef struct { void *hvc6_userdata; } guest_t; -/* Bump the page table generation counter (call after any PT modification). This - * invalidates all per-thread GVA TLB caches. +/* Bump the page table generation counter (call after any host PT modification). + * This invalidates all per-thread GVA TLB caches. */ static inline void guest_pt_gen_bump(guest_t *g) { @@ -685,8 +745,8 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) hv_vcpu_set_reg(vcpu, HV_REG_X11, cpu_tlbi_req.icache_flush ? 1 : 0); break; case TLBI_RANGE_LARGE: { - /* Single-shot TLBI RVAE1IS for ranges in (16..64] pages. The operand - * format and the SCALE=0 / TG=01 / ASID=0 assumptions are documented at + /* Single-shot TLBI RVAE1IS for ranges above the selective cap. The + * SCALE/NUM format and TG=01 assumption are documented at * tlbi_rvae1is_operand above. ASID stays 0 because the shim runs * single-ASID (TCR_EL1.A1=0, TTBR0 ASID=0; rosetta does not allocate a * separate ASID). If a future change introduces non-zero ASIDs, the @@ -708,6 +768,31 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) tlbi_request_clear(); } +/* RVAE1IS requires BaseADDR alignment to its SCALE granule. Widen a requested + * interval to that granule, repeating if widening crosses a SCALE threshold. + * Over-invalidation is architecturally harmless and preserves unrelated TLB + * entries far better than a VMALLE1IS broadcast. + */ +static inline bool tlbi_rvae_normalize(uint64_t *start, uint64_t *end) +{ + for (int pass = 0; pass < 3; pass++) { + uint64_t pages = (*end - *start) >> 12; + if (pages <= TLBI_SELECTIVE_MAX_PAGES) + return true; + uint64_t unit_pages = tlbi_rvae_unit_pages(pages); + uint64_t unit_bytes = unit_pages << 12; + uint64_t s = *start & ~(unit_bytes - 1); + if (*end > UINT64_MAX - (unit_bytes - 1)) + return false; + uint64_t e = (*end + unit_bytes - 1) & ~(unit_bytes - 1); + *start = s; + *end = e; + if (((e - s) >> 12) <= unit_pages * 32) + return ((e - s) >> 12) <= TLBI_RVAE_MAX_PAGES; + } + return false; +} + static inline void tlbi_request_range(uint64_t start, uint64_t end) { if (cpu_tlbi_req.kind == TLBI_BROADCAST) @@ -737,6 +822,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) */ uint64_t large_cap = g_tlbi_range_supported ? TLBI_RVAE_MAX_PAGES : TLBI_SELECTIVE_MAX_PAGES; + if (g_tlbi_range_supported && n > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&s, &e)) { + tlbi_request_broadcast(); + return; + } + n = (e - s) >> 12; + } if (n > large_cap) { tlbi_request_broadcast(); return; @@ -768,6 +860,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) uint64_t us = s < es ? s : es; uint64_t ue = e > ee ? e : ee; uint64_t un = (ue - us) >> 12; + if (g_tlbi_range_supported && un > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&us, &ue)) { + tlbi_request_broadcast(); + return; + } + un = (ue - us) >> 12; + } if (un > large_cap) { tlbi_request_broadcast(); return; @@ -1019,11 +1118,22 @@ int guest_install_va_pages(guest_t *g, uint64_t gpa, int perms); -/* Query whether a 2 MiB TTBR0 VA block already has a leaf mapping. - * Returns true only for a present L2 block descriptor. +/* Query whether a 2 MiB TTBR0 VA block already has any L2 entry, either a block + * descriptor or an L3 table descriptor. */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); +/* Rebuild the low-VA PTE occupancy index from TTBR0. Used after fork restores + * page-table pages into a freshly initialized guest_t. + */ +void guest_rebuild_pte_present(guest_t *g); + +/* Reconcile the host-only 2 MiB occupancy index after EL1 has invalidated + * descriptors in [start,end). This observes PTEs only; it never writes a + * descriptor or requests another TLBI. Caller holds mmap_lock. + */ +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end); + /* Returns true when the VA range [va, va+size) overlaps the user-VA kbuf alias * window [KBUF_USER_VA, KBUF_USER_VA+KBUF_SIZE). Callers that install TTBR0 * mappings (the future rosetta_finalize, sys_mmap MAP_FIXED touching this @@ -1051,6 +1161,36 @@ static inline bool guest_kbuf_user_va_overlap(uint64_t va, uint64_t size) */ void *guest_ptr(const guest_t *g, uint64_t gva); +/* Like guest_ptr_avail but never triggers lazy fault-in. For callers that + * already hold mmap_lock or must avoid acquiring it. + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms); + +/* Materialize lazy blocks intersecting [gva, gva+len). Takes mmap_lock; call + * only without locks that nest below it. Later access under those locks must + * use nofault helpers: pre-faulting neither pins mappings nor guarantees + * success. + * + * Returns 0 if anything was (or already is) materialized, -1 otherwise; callers + * that merely pre-fault can ignore the result. + */ +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); + +/* Materialization with mmap_lock already held. Follow with nofault access. */ +int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); + +/* Smallest 2MiB-block-aligned va' in [va, end) containing at least one valid + * TTBR0 PTE, or end if none. Low VA uses the host-side hierarchical occupancy + * bitmap; non-identity/high VA falls back to the page-table hierarchy. Locking: + * callers MUST hold mmap_lock. + */ +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end); + /* Get a host pointer for a guest virtual address (write access). * Returns NULL if gva is out of bounds or not writable. Same validity window as * guest_ptr. @@ -1086,6 +1226,9 @@ void *guest_ptr_bound(const guest_t *g, */ int guest_read(const guest_t *g, uint64_t gva, void *dst, size_t len); +/* Guest-to-host copy without lazy materialization; a missing PTE fails. */ +int guest_read_nofault(const guest_t *g, uint64_t gva, void *dst, size_t len); + /* Optimized guest-to-host copy for small fixed-size inputs. Uses a direct guest * pointer when the full range is contiguous and readable, otherwise falls back * to guest_read() for boundary-crossing safety. @@ -1110,6 +1253,16 @@ size_t guest_write_partial(guest_t *g, uint64_t gva, const void *src, size_t len); +size_t guest_write_partial_nofault(guest_t *g, + uint64_t gva, + const void *src, + size_t len); + +/* Same copy without lazy materialization. Callers that already hold mmap_lock + * can use this after guest_lazy_faultin_locked(); an invalid destination then + * fails instead of recursively trying to acquire mmap_lock. + */ +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len); /* Optimized host-to-guest copy for small fixed-size outputs. Uses a direct * guest pointer when the full range is contiguous and writable, otherwise falls @@ -1385,12 +1538,33 @@ bool guest_region_range_has_ro_shared_backing(const guest_t *g, uint64_t start, uint64_t end); -/* Try to materialize a lazy (MAP_NORESERVE) page at the given offset. Called - * from the data/instruction abort handler when the faulting address falls - * within a noreserve region. Creates page table entries for one 2MiB block - * containing the fault address, zeros the memory, and clears the noreserve flag - * for the materialized sub-range. +/* Try to materialize a lazy (deferred-PTE: private anonymous or MAP_NORESERVE) + * page at the given offset. Called from the data/instruction abort handler when + * the faulting address falls within a lazy region, and from the host-access + * fault-in path when a syscall targets a lazy range the guest has not touched + * yet. Creates page table entries for one 2MiB block containing the fault + * address. A slab block known to be clean skips zeroing; a possibly-dirty block + * zeros invalid pages before publishing them. A block that is already valid + * returns success without re-zeroing (concurrent-fault idempotence). * Returns 0 on success (caller should TLBI and retry), -1 if the offset is not - * in a noreserve region. + * in a lazy region or the region is PROT_NONE. Locking: callers MUST hold + * mmap_lock. */ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset); + +/* Guest-fault variant with per-vCPU sequential fault-around. */ +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset); + +/* Dirty-map and in-flight-claim helpers. Callers hold mmap_lock. */ +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start); +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end); +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end); +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end); +void guest_materialize_wait_all_locked(guest_t *g); + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Locking: + * callers MUST hold mmap_lock. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va); diff --git a/src/core/launch.c b/src/core/launch.c index bbe068b7..5b8a195b 100644 --- a/src/core/launch.c +++ b/src/core/launch.c @@ -26,6 +26,7 @@ #include "core/bootstrap.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/sysroot.h" @@ -192,6 +193,14 @@ int elfuse_launch(const launch_args_t *args) 0) goto fail; + /* Tracing/debuggers require every mmap to reach host dispatch so syscall + * observation and region state advance in lockstep. This also prevents a + * trace-gated shim from leaving invisible arena reservations that perturb + * the host slow path's address-hint behavior. + */ + if (args->verbose || args->gdb_port > 0) + mmap_fastpath_disable(&g); + /* GDB setup must happen before the first run so entry-stop and hardware * breakpoints can affect the initial vCPU. */ diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h new file mode 100644 index 00000000..a0e2be7b --- /dev/null +++ b/src/core/mmap-fastpath.h @@ -0,0 +1,221 @@ +/* + * Per-vCPU EL1 anonymous-mmap consumer rings. + * + * The host produces arenas, consumes mmap/munmap publications, and returns + * metadata-committed holes through a reverse SPSC ring. EL1 first-fit consumes + * those private extents before bump-allocating fresh VA. Control blocks live in + * the EL1-only shim-data mapping and are selected from SP_EL1's per-thread + * stack slot, so no guest-visible register ABI is consumed. + */ + +#pragma once + +#include +#include +#include +#include + +#include "core/guest.h" +#include "proved/mmap-fastpath.h" + +typedef struct thread_entry thread_entry_t; + +#define SHIM_MMAP_CONTROL_BASE 0x20000u +#define SHIM_MMAP_CONTROL_STRIDE 0x1000u +#define SHIM_MMAP_RING_SIZE 32u +#define SHIM_MMAP_CTRL_ENABLED 0x1u +#define SHIM_MMAP_CTRL_TLBIRANGE 0x2u + +/* Host page-table writers set this gate before changing an arena descriptor or + * a stage-1 PTE. Each EL1 producer announces itself in its private control + * before rechecking the gate. This is a writer-vs-per-vCPU-reader handshake, + * not an allocator lock: fast munmap producers never write a shared cache line + * or wait for one another. + */ +#define SHIM_MMAP_PT_GATE_OFF 0x2188u + +/* EL1 munmap advances this once its descriptor stores and TLBI have completed + * and before it releases the retire tail. guest_t.pt_gen counts host writers + * only, so gva_translate_perm keys its cache on both words: without this one a + * host thread that is not a vCPU thread, and so never drains at a VM exit, + * keeps serving a retired range until some thread takes mmap_lock. One shared + * line per PTE-clearing retirement, beside the broadcast TLBI it already pays. + */ +#define SHIM_MMAP_PT_EPOCH_OFF 0x2190u + +#define SHIM_MUNMAP_RETIRE_RING_SIZE 32u +#define SHIM_MUNMAP_RETIRE_OFF 0x400u +#define SHIM_MUNMAP_RETIRE_BYTES_SOFT (256ULL * 1024 * 1024) +#define SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK 0x3fu +#define SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT 6u +#define SHIM_MUNMAP_RETIRE_F_CHARGE_MASK 0xffffffc0u + +/* MMAP_FAST_ARENA_MIN/MAX/TARGET_ENTRIES and MMAP_FAST_PUBLICATION_WINDOW are + * defined in proved/mmap-fastpath.h, included above: that header's sizing + * arithmetic is proved against those exact values, so this is the one + * definition rather than a copy a proof cannot see drift from. + * + * MMAP_FAST_PUBLICATION_WINDOW must stay a power of two: the sequence counter + * indexes the window by masking. + */ + +enum { + SHIM_MMAP_COUNTER_SHAPE_MISS = 0, + SHIM_MMAP_COUNTER_CAPACITY_MISS, + SHIM_MMAP_COUNTER_RING_FULL, + SHIM_MMAP_COUNTER_GENERATION_STALE, + SHIM_MMAP_COUNTER_ATTENTION, + SHIM_MMAP_COUNTER_HIT, + SHIM_MMAP_COUNTERS_N, +}; + +typedef struct { + uint64_t addr; + uint64_t len; + uint64_t prot; +} shim_mmap_entry_t; + +typedef struct munmap_retire_entry { + uint64_t addr; + uint64_t length; + uint32_t arena_generation; + uint32_t flags; +} munmap_retire_entry_t; + +typedef struct munmap_retire_ring { + _Atomic uint32_t head; /* host consumer */ + _Atomic uint32_t tail; /* EL1 producer */ + /* Monotonic accounting of conservative materialized-byte upper bounds. */ + _Atomic uint64_t produced_bytes; /* EL1 producer */ + _Atomic uint64_t consumed_bytes; /* host consumer */ + _Atomic uint32_t producer_active; + /* Advisory; never forces the producer to exit. */ + _Atomic uint32_t cleanup_requested; + munmap_retire_entry_t entries[SHIM_MUNMAP_RETIRE_RING_SIZE]; +} munmap_retire_ring_t; + + +typedef struct { + _Atomic uint32_t generation; /* host publish word */ + _Atomic uint32_t consumer_generation; /* EL1 generation ack */ + _Atomic uint32_t flags; /* host-owned enable bits */ + _Atomic uint32_t head; /* host consumer cursor */ + _Atomic uint32_t tail; /* EL1 producer cursor */ + uint32_t _pad0; + _Atomic uint64_t arena_base; + _Atomic uint64_t arena_limit; + _Atomic uint64_t cursor; /* EL1 bump cursor */ + uint64_t next_arena_size; /* most recently selected generation size */ + uint64_t publication_seq; /* host-only: registrations, rotates the window */ + shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; + _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; + uint64_t refill_count; + uint64_t recycle_count; + uint64_t peak_arena_size; + _Atomic uint32_t materialized_generation; + uint32_t _pad1; + _Atomic uint64_t materialized_start; + _Atomic uint64_t materialized_end; + uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; + munmap_retire_ring_t retire; + + /* Host-only refill policy history; see MMAP_FAST_PUBLICATION_WINDOW. Kept + * past the rings so adding it moves no offset the EL1 side depends on. + */ + uint64_t publication_window[MMAP_FAST_PUBLICATION_WINDOW]; + + /* EL1-incremented, host-read: how many munmap() calls found the retirement + * ring near full (see el1_munmap's near-full check) and fell back to a + * synchronous host trap instead of publishing into the ring. Placed past + * every fixed EL1 ABI offset, like publication_window above. + */ + _Atomic uint64_t munmap_retire_near_full; + + /* EL1 sets this after publishing mmap or munmap work. The host uses it only + * to avoid an empty opportunistic VM-exit drain; lock-taking paths still + * drain unconditionally. + */ + _Atomic uint32_t pending_work; + + /* EL1 consumes before publishing; host replenishes while this producer is + * stopped, either in HVC or behind the closed gate. metadata_reserved + * includes credits and undrained publications and is owned by the host + * under mmap_lock. + */ + _Atomic uint32_t metadata_credits; + uint32_t metadata_reserved; +} shim_mmap_control_t; + +_Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, + "EL1 retire-ring offset ABI"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "mmap control exceeds per-vCPU stride"); + +/* Provision the main vCPU before guest entry. Worker vCPUs provision lazily on + * their first eligible mmap so short-lived threads do not allocate an unused + * arena. + */ +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); + +/* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ +void mmap_fastpath_drain_locked(guest_t *g); + +/* Opportunistically drain publications and retirements at a natural VM exit. + * Safe before every exit handler; it acquires mmap_lock internally. When a + * fork-family syscall is pending, skip the arena top-up that the syscall will + * immediately revoke. + */ +void mmap_fastpath_drain_vmexit(guest_t *g, bool fork_family_pending); + +/* True when the stopped current vCPU was interrupted in the middle of its EL1 + * producer critical section. A cancellation exit must resume it before host + * drain can close the PT gate. + */ +bool mmap_fastpath_current_producer_active(const guest_t *g); + +/* Mark every arena intersecting a successfully materialized lazy range. Caller + * holds mmap_lock. EL1 may skip its PTE walk only while this marker differs + * from the arena's current generation. + */ +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end); + +/* Refill the current vCPU after an eligible mmap slow-path. request_len is + * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. + * Caller holds mmap_lock. + */ +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); + +/* Fulfil an eligible mmap that reached HVC (capacity/ring/generation fallback) + * directly from the current vCPU's refilled arena. Metadata is committed by the + * host immediately, so no mmap publication entry is needed. Caller holds + * mmap_lock. + */ +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); + +/* Give an explicit slow-path hint precedence over this stopped vCPU's + * unconsumed arena tail. Caller holds mmap_lock. + */ +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length); + +/* Revoke all arenas while sibling vCPUs are quiesced. Caller holds mmap_lock. + */ +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); + +/* Disable the feature before first guest entry (debugger/observability). */ +void mmap_fastpath_disable(guest_t *g); + +/* Advance *start past an EL1 arena reservation that overlaps length bytes. */ +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr); diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 91846235..d0aca752 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -19,6 +20,7 @@ #include "hvutil.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/vdso.h" #include "debug/log.h" @@ -86,12 +88,12 @@ _Static_assert((SHIM_COUNTERS_OFF & 0xFFF) == 0xC8, "shim.S SHIM_COUNTERS_OFF_LO12 hard-coded to 0xC8"); _Static_assert((SHIM_COUNTERS_OFF & ~0xFFF) == 0x1000, "shim.S SHIM_COUNTERS_OFF_HI hard-coded to 0x1000"); -_Static_assert(SHIM_IDENTITY_OFF_PGID == 0x1158, - "shim.S getpgid fast path hard-codes PGID off 0x1158"); -_Static_assert(SHIM_IDENTITY_OFF_SID == 0x1160, - "shim.S getsid fast path hard-codes SID off 0x1160"); -_Static_assert(SHIM_FUTEX_WAITERS_OFF == 0x1168, - "shim.S futex_wake_fast hard-codes the waiter array at 0x1168"); +_Static_assert(SHIM_IDENTITY_OFF_PGID == 0x1178, + "shim.S getpgid fast path hard-codes PGID off 0x1178"); +_Static_assert(SHIM_IDENTITY_OFF_SID == 0x1180, + "shim.S getsid fast path hard-codes SID off 0x1180"); +_Static_assert(SHIM_FUTEX_WAITERS_OFF == 0x1188, + "shim.S futex_wake_fast hard-codes the waiter array at 0x1188"); _Static_assert(SHIM_IDENTITY_OFF_SID + 8 <= SHIM_FUTEX_WAITERS_OFF, "waiter array must not overlap the SID slot"); _Static_assert(SHIM_GLOBALS_SIZE >= SHIM_IDENTITY_OFF_SID + 8, @@ -101,6 +103,90 @@ _Static_assert(SHIM_GLOBALS_SIZE <= BLOCK_2MIB, _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= SHIM_IDENTITY_OFF_PGID, "counter array must not overlap the PGID slot"); +_Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, + "EL1 mmap fast path hard-codes control base 0x20000"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x1000, + "EL1 mmap fast path hard-codes control stride 0x1000"); +_Static_assert(SHIM_MMAP_RING_SIZE == 32, + "EL1 mmap fast path hard-codes 32 ring entries"); +_Static_assert(offsetof(shim_mmap_control_t, generation) == 0, + "EL1 mmap generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, + "EL1 mmap consumer-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, flags) == 8, + "EL1 mmap flags offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, head) == 12, + "EL1 mmap head offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, tail) == 16, + "EL1 mmap tail offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_base) == 24, + "EL1 mmap arena-base offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_limit) == 32, + "EL1 mmap arena-limit offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, cursor) == 40, + "EL1 mmap cursor offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, next_arena_size) == 48, + "mmap next-arena-size offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, publication_seq) == 56, + "mmap publication-seq offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, ring) == 64, + "EL1 mmap ring offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x340, + "EL1 mmap counter offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_generation) == 0x388, + "EL1 mmap materialized-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_start) == 0x390 && + offsetof(shim_mmap_control_t, materialized_end) == 0x398, + "EL1 mmap materialized bounds offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, retire) == 0x400, + "EL1 munmap retire offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, produced_bytes) == 8, + "EL1 munmap produced-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, consumed_bytes) == 16, + "EL1 munmap consumed-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, producer_active) == 24, + "EL1 munmap active offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, cleanup_requested) == 28, + "EL1 munmap cleanup-request offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, entries) == 32, + "EL1 munmap entries offset drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_RING_SIZE == 32 && MAX_THREADS == 64, + "EL1 munmap ring/arena scan constants drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_BYTES_SOFT == 0x10000000ULL, + "EL1 munmap byte advisory threshold drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK == 0x3f && + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT == 6, + "EL1 munmap retire flag encoding drift"); + +/* Host-only, so no EL1 offset depends on it; the assert pins it past every ring + * instead, which is what keeps adding it from moving the shared layout. + */ +_Static_assert(offsetof(shim_mmap_control_t, publication_window) > + offsetof(shim_mmap_control_t, retire), + "publication window must not precede the EL1-visible rings"); +_Static_assert((MMAP_FAST_PUBLICATION_WINDOW & + (MMAP_FAST_PUBLICATION_WINDOW - 1)) == 0, + "publication window size must be a power of two"); +_Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= + (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), + "munmap retire flags cannot encode maximum arena charge"); +_Static_assert(SHIM_MMAP_PT_GATE_OFF >= SHIM_GLOBALS_SIZE && + SHIM_MMAP_PT_GATE_OFF + sizeof(uint32_t) <= + SHIM_MMAP_CONTROL_BASE, + "host PT gate overlaps shim globals or mmap controls"); +_Static_assert(SHIM_MMAP_PT_EPOCH_OFF >= + SHIM_MMAP_PT_GATE_OFF + sizeof(uint32_t) && + SHIM_MMAP_PT_EPOCH_OFF % sizeof(uint64_t) == 0 && + SHIM_MMAP_PT_EPOCH_OFF + sizeof(uint64_t) <= + SHIM_MMAP_CONTROL_BASE, + "EL1 PT epoch overlaps the PT gate or mmap controls"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "per-vCPU mmap control exceeds its shim-data stride"); +_Static_assert(SHIM_MMAP_CONTROL_BASE + + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE <= + BLOCK_2MIB - MAX_THREADS * 4096, + "mmap controls overlap per-vCPU EL1 stacks"); static uint8_t *cache_base(const guest_t *g) { @@ -131,7 +217,13 @@ static void urandom_ring_unlock(_Atomic uint32_t *lock_p) void shim_globals_init(guest_t *g) { - memset(cache_base(g), 0, SHIM_GLOBALS_SIZE); + /* mmap controls occupy a separate low shim-data range. Init/exec/fork child + * all call this while no sibling can execute, so clearing the whole control + * array also prevents a recycled SP_EL1 slot from inheriting an arena + * published to its previous owner. + */ + memset(cache_base(g), 0, + SHIM_MMAP_CONTROL_BASE + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE); } void shim_globals_publish_pid(guest_t *g, int64_t pid, int64_t ppid) @@ -459,7 +551,7 @@ _Static_assert(SHIM_COUNTER_URANDOM_HIT * 8 == 72, "CB_URANDOM_HIT"); _Static_assert(SHIM_COUNTER_GETRANDOM_HIT * 8 == 80, "CB_GETRANDOM_HIT"); _Static_assert(SHIM_COUNTER_PGSID_HIT * 8 == 88, "CB_PGSID_HIT"); _Static_assert(SHIM_COUNTER_FUTEX_EAGAIN_HIT * 8 == 96, "CB_FUTEX_EAGAIN_HIT"); -_Static_assert(SHIM_COUNTER_FUTEX_EFAULT_HIT * 8 == 104, "CB_FUTEX_EFAULT_HIT"); +_Static_assert(SHIM_COUNTER_FUTEX_FAULT_BAIL * 8 == 104, "CB_FUTEX_FAULT_BAIL"); _Static_assert(SHIM_COUNTER_FUTEX_SHAPE_BAIL * 8 == 112, "CB_FUTEX_SHAPE_BAIL"); _Static_assert(SHIM_COUNTER_FUTEX_MATCH_BAIL * 8 == 120, "CB_FUTEX_MATCH_BAIL"); _Static_assert(SHIM_COUNTER_FUTEX_WAKE_HIT * 8 == 128, "CB_FUTEX_WAKE_HIT"); @@ -480,17 +572,15 @@ static const char *const counter_names[SHIM_COUNTERS_N] = { [SHIM_COUNTER_GETRANDOM_HIT] = "GETRANDOM_HIT", [SHIM_COUNTER_PGSID_HIT] = "PGSID_HIT", [SHIM_COUNTER_FUTEX_EAGAIN_HIT] = "FUTEX_EAGAIN_HIT", - [SHIM_COUNTER_FUTEX_EFAULT_HIT] = "FUTEX_EFAULT_HIT", + [SHIM_COUNTER_FUTEX_FAULT_BAIL] = "FUTEX_FAULT_BAIL", [SHIM_COUNTER_FUTEX_SHAPE_BAIL] = "FUTEX_SHAPE_BAIL", [SHIM_COUNTER_FUTEX_MATCH_BAIL] = "FUTEX_MATCH_BAIL", [SHIM_COUNTER_FUTEX_WAKE_HIT] = "FUTEX_WAKE_HIT", [SHIM_COUNTER_FUTEX_WAKE_WAITER_BAIL] = "FUTEX_WAKE_WAITER_BAIL", - - /* Every slot of SHIM_COUNTERS_N == 18 is now named. The dump still prints - * "(reserved)" for an unnamed slot, which would flag an out-of-band - * increment; a future EL1 service needs the array grown rather than a free - * slot claimed. - */ + [SHIM_COUNTER_FAULT_MATERIALIZE] = "FAULT_MATERIALIZE", + [SHIM_COUNTER_FAULT_TLBI_VAE] = "FAULT_TLBI_VAE", + [SHIM_COUNTER_FAULT_TLBI_RVAE] = "FAULT_TLBI_RVAE", + [SHIM_COUNTER_FAULT_TLBI_BCAST] = "FAULT_TLBI_BCAST", }; uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) @@ -503,6 +593,16 @@ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) return atomic_load_explicit(slot_p, memory_order_relaxed); } +void shim_globals_counter_inc(guest_t *g, unsigned slot) +{ + if (!shim_globals_stats_enabled() || slot >= SHIM_COUNTERS_N) + return; + uint8_t *page = (uint8_t *) g->host_base + g->shim_data_base; + _Atomic uint64_t *slot_p = + (_Atomic uint64_t *) (page + SHIM_COUNTERS_OFF) + slot; + atomic_fetch_add_explicit(slot_p, 1, memory_order_relaxed); +} + void shim_globals_counters_dump(const guest_t *g) { fprintf(stderr, "shim-stats (pid=%lld)\n", (long long) proc_get_pid()); @@ -514,6 +614,67 @@ void shim_globals_counters_dump(const guest_t *g) fprintf(stderr, " %-20s %llu\n", name ? name : "(reserved)", (unsigned long long) v); } + + static const char *const mmap_counter_names[SHIM_MMAP_COUNTERS_N] = { + [SHIM_MMAP_COUNTER_SHAPE_MISS] = "MMAP_SHAPE_MISS", + [SHIM_MMAP_COUNTER_CAPACITY_MISS] = "MMAP_CAPACITY_MISS", + [SHIM_MMAP_COUNTER_RING_FULL] = "MMAP_RING_FULL", + [SHIM_MMAP_COUNTER_GENERATION_STALE] = "MMAP_GENERATION_STALE", + [SHIM_MMAP_COUNTER_ATTENTION] = "MMAP_ATTENTION", + [SHIM_MMAP_COUNTER_HIT] = "MMAP_HIT", + }; + uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; + uint64_t refill_count = 0, recycle_count = 0; + uint64_t current_max = 0, peak_max = 0; + uint64_t munmap_retire_near_full = 0; + const uint8_t *shim_data = + (const uint8_t *) g->host_base + g->shim_data_base; + for (int slot = 0; slot < MAX_THREADS; slot++) { + const shim_mmap_control_t *c = + (const shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + mmap_counters[i] += + atomic_load_explicit(&c->counters[i], memory_order_relaxed); + refill_count += c->refill_count; + recycle_count += c->recycle_count; + if (c->next_arena_size > current_max) + current_max = c->next_arena_size; + if (c->peak_arena_size > peak_max) + peak_max = c->peak_arena_size; + munmap_retire_near_full += atomic_load_explicit( + &c->munmap_retire_near_full, memory_order_relaxed); + } + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], + (unsigned long long) mmap_counters[i]); + fprintf(stderr, " %-20s %llu\n", "MMAP_REFILL", + (unsigned long long) refill_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", + (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MUNMAP_RETIRE_NEAR_FULL", + (unsigned long long) munmap_retire_near_full); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", + (unsigned long long) current_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", + (unsigned long long) peak_max); + uint64_t high_water = + g->mmap_next > MMAP_BASE ? g->mmap_next - MMAP_BASE : 0; + fprintf(stderr, " %-20s %llu\n", "MMAP_HIGH_WATER", + (unsigned long long) high_water); + fprintf(stderr, " %-20s %llu\n", "FAULT_CLEAN_SKIP", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP]); + fprintf(stderr, " %-20s %llu\n", "FAULT_DIRTY_MEMSET", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_DIRTY_MEMSET]); + fprintf(stderr, " %-20s %llu\n", "FAULT_ALREADY_VALID", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]); + fprintf(stderr, " %-20s %llu\n", "FAULT_WINDOW_BYTES", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES]); } static pthread_once_t stats_once = PTHREAD_ONCE_INIT; diff --git a/src/core/shim-globals.h b/src/core/shim-globals.h index 08e593a7..ae13ba44 100644 --- a/src/core/shim-globals.h +++ b/src/core/shim-globals.h @@ -127,11 +127,12 @@ /* Fast-path hit / miss counters. * - * 18 uint64 slots after the ring lock, bumped by identity_class_fast, + * 22 uint64 slots after the ring lock, bumped by identity_class_fast, * urandom_read_fast, futex_wait_fast and futex_wake_fast at every hit and bail * so fast-path activity can be attributed rather than guessed. Slots 0-7 are - * bail reasons, 8-11 identity and urandom hits, 12-13 futex wait hits, 14-15 - * futex wait bails, 16-17 the futex wake path's hit and its one bail. + * bail reasons, 8-11 identity and urandom hits, 12 the futex wait hit, 13-15 + * futex wait bails, 16-17 the futex wake path's hit and its one bail, and 18-21 + * lazy-fault materializations and the TLBI wire mode they emit. * * Increments are plain load-add-store, so concurrent bails on separate vCPUs * lose a few. These are diagnostic ratios, not accounting. @@ -146,7 +147,7 @@ * keep the two in sync. */ #define SHIM_COUNTERS_OFF 0x10C8 -#define SHIM_COUNTERS_N 18 +#define SHIM_COUNTERS_N 22 #define SHIM_COUNTER_ATTN_BAIL 0 #define SHIM_COUNTER_URANDOM_FD_OOR 1 @@ -161,11 +162,15 @@ #define SHIM_COUNTER_GETRANDOM_HIT 10 #define SHIM_COUNTER_PGSID_HIT 11 #define SHIM_COUNTER_FUTEX_EAGAIN_HIT 12 -#define SHIM_COUNTER_FUTEX_EFAULT_HIT 13 +#define SHIM_COUNTER_FUTEX_FAULT_BAIL 13 #define SHIM_COUNTER_FUTEX_SHAPE_BAIL 14 #define SHIM_COUNTER_FUTEX_MATCH_BAIL 15 #define SHIM_COUNTER_FUTEX_WAKE_HIT 16 #define SHIM_COUNTER_FUTEX_WAKE_WAITER_BAIL 17 +#define SHIM_COUNTER_FAULT_MATERIALIZE 18 +#define SHIM_COUNTER_FAULT_TLBI_VAE 19 +#define SHIM_COUNTER_FAULT_TLBI_RVAE 20 +#define SHIM_COUNTER_FAULT_TLBI_BCAST 21 /* Extended identity slots: pgid and sid. * @@ -174,8 +179,8 @@ * matches. The host re-publishes after setpgid / setsid / exec / fork so the * slots match guest_pgid / guest_sid in proc-identity.c. */ -#define SHIM_IDENTITY_OFF_PGID 0x1158 -#define SHIM_IDENTITY_OFF_SID 0x1160 +#define SHIM_IDENTITY_OFF_PGID 0x1178 +#define SHIM_IDENTITY_OFF_SID 0x1180 /* Per-bucket futex waiter counts, one uint32 per hash bucket of the table in * runtime/futex.c. The shim's wake fast path reads one of these to answer a @@ -200,7 +205,7 @@ * the table size are pinned by static asserts in runtime/futex.c. */ #define SHIM_FUTEX_BUCKETS 1024u -#define SHIM_FUTEX_WAITERS_OFF 0x1168 +#define SHIM_FUTEX_WAITERS_OFF 0x1188 #define SHIM_FUTEX_WAITERS_BYTES (SHIM_FUTEX_BUCKETS * 4u) #define SHIM_GLOBALS_SIZE (SHIM_FUTEX_WAITERS_OFF + SHIM_FUTEX_WAITERS_BYTES) @@ -374,6 +379,7 @@ void shim_globals_refill_urandom_ring(guest_t *g); * ELFUSE_SHIM_STATS. */ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot); +void shim_globals_counter_inc(guest_t *g, unsigned slot); void shim_globals_counters_dump(const guest_t *g); /* ELFUSE_SHIM_STATS env-var gate (idempotent / cached). When enabled the exit diff --git a/src/core/shim-mmap.c b/src/core/shim-mmap.c new file mode 100644 index 00000000..7e74dc6c --- /dev/null +++ b/src/core/shim-mmap.c @@ -0,0 +1,969 @@ +/* + * EL1 mmap-family syscall fast paths. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * This file is compiled freestanding and linked into the shim image. It may not + * call host code or a C runtime: the only state it consumes is the saved EL0 + * register frame, TPIDR_EL1's shim-data mapping, TTBR0_EL1, and the shared mmap + * control protocol. Keep architecture-only operations in the small helpers + * below; the synchronization and allocator policy remain ordinary C. + */ + +#include "core/shim-mmap.h" + +#include +#include +#include + +#include "core/mmap-fastpath.h" +#include "core/shim-globals.h" + +#define EL1_PAGE_SIZE 0x1000ULL +#define EL1_BLOCK_SIZE 0x200000ULL +#define EL1_WINDOW_1G 0x40000000ULL +#define EL1_SHIM_DATA_SIZE 0x200000ULL + +#define EL1_SYS_MUNMAP 215ULL +#define EL1_SYS_MMAP 222ULL + +#define EL1_PROT_READ 1ULL +#define EL1_PROT_WRITE 2ULL +#define EL1_PROT_RW (EL1_PROT_READ | EL1_PROT_WRITE) + +#define EL1_MAP_PRIVATE 0x02ULL +#define EL1_MAP_ANONYMOUS 0x20ULL +#define EL1_MAP_NORESERVE 0x4000ULL + +#define EL1_DESC_ADDR_MASK 0x0000fffffffff000ULL +#define EL1_DESC_TYPE_MASK 0x3ULL +#define EL1_DESC_BLOCK 0x1ULL +#define EL1_DESC_TABLE 0x3ULL + +#define EL1_L3_PTES_PER_BATCH 8u +#define EL1_TLBI_LEVEL_L2 2u +#define EL1_TLBI_LEVEL_L3 3u + +/* The bump path promises 2 MiB-aligned starts at or above this request size, so + * the host can back them with L2 blocks. + */ +#define EL1_ALIGN_THRESHOLD EL1_BLOCK_SIZE + +/* Arena descriptors are one per vCPU slot. shim-globals.c pins this against + * MAX_THREADS; the host thread header is not includable freestanding. + */ +#define EL1_ARENA_SLOTS 64u + +typedef struct { + uint8_t *shim_data; + shim_mmap_control_t *control; + _Atomic uint32_t *gate; + _Atomic uint64_t *pt_epoch; + unsigned slot; +} el1_mmap_context_t; + +typedef enum { + EL1_GATE_ENTERED, + EL1_GATE_CLOSED_BEFORE_ANNOUNCE, + EL1_GATE_CLOSED_AFTER_ANNOUNCE, +} el1_gate_result_t; + +_Static_assert(EL1_SAVED_GPRS * sizeof(uint64_t) == 248, + "saved GPR frame layout changed"); +_Static_assert(sizeof(shim_mmap_entry_t) == 24, + "EL1 mmap publication ABI changed"); +_Static_assert(sizeof(munmap_retire_entry_t) == 24, + "EL1 retire publication ABI changed"); + +static inline uint64_t el1_read_tpidr(void) +{ + uint64_t value; + __asm__ volatile("mrs %0, tpidr_el1" : "=r"(value)); + return value; +} + +static inline uint64_t el1_read_ttbr0(void) +{ + uint64_t value; + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(value)); + return value; +} + +static inline void el1_dsb_ishst(void) +{ + __asm__ volatile("dsb ishst" ::: "memory"); +} + +static inline void el1_dsb_ish(void) +{ + __asm__ volatile("dsb ish" ::: "memory"); +} + +static inline void el1_isb(void) +{ + __asm__ volatile("isb" ::: "memory"); +} + +static inline void el1_tlbi_vale1is(uint64_t operand) +{ + __asm__ volatile("tlbi vale1is, %0" : : "r"(operand) : "memory"); +} + +static inline void el1_tlbi_rvale1is(uint64_t operand) +{ + __asm__ volatile("tlbi rvale1is, %0" : : "r"(operand) : "memory"); +} + +static inline void el1_tlbi_all(void) +{ + __asm__ volatile("tlbi vmalle1is" ::: "memory"); +} + +static inline uint64_t el1_desc_address(uint64_t descriptor) +{ + return descriptor & EL1_DESC_ADDR_MASK; +} + +static inline uint64_t el1_desc_type(uint64_t descriptor) +{ + return descriptor & EL1_DESC_TYPE_MASK; +} + +/* Guest VA is identity-mapped over the arena, so a descriptor's output address + * is also the VA of the next table. + */ +static inline _Atomic uint64_t *el1_table(uint64_t descriptor) +{ + return (_Atomic uint64_t *) (uintptr_t) el1_desc_address(descriptor); +} + +static inline uint64_t el1_block_start(uint64_t address) +{ + return address & ~(EL1_BLOCK_SIZE - 1); +} + +static inline uint64_t el1_next_block(uint64_t address) +{ + return el1_block_start(address) + EL1_BLOCK_SIZE; +} + +static inline uint64_t el1_min(uint64_t left, uint64_t right) +{ + return left < right ? left : right; +} + +static inline bool el1_add_overflow(uint64_t left, + uint64_t right, + uint64_t *result) +{ + return __builtin_add_overflow(left, right, result); +} + +static bool el1_page_round(uint64_t length, uint64_t *rounded) +{ + uint64_t value; + if (length == 0 || el1_add_overflow(length, EL1_PAGE_SIZE - 1, &value)) + return false; + *rounded = value & ~(EL1_PAGE_SIZE - 1); + return true; +} + +/* SP_EL1 stack tops are shim_data_end - slot*4KiB and controls are shim_data + + * 0x20000 + slot*4KiB, so the saved frame's page locates the control without + * consuming another system register. + */ +static el1_mmap_context_t el1_context(uint64_t *saved_gprs) +{ + uintptr_t shim_data = (uintptr_t) el1_read_tpidr(); + uintptr_t frame = (uintptr_t) saved_gprs; + uintptr_t stack_top = + (frame + EL1_PAGE_SIZE - 1) & ~(uintptr_t) (EL1_PAGE_SIZE - 1); + uintptr_t slot_bytes = shim_data + EL1_SHIM_DATA_SIZE - stack_top; + unsigned slot = (unsigned) (slot_bytes / EL1_PAGE_SIZE); + return (el1_mmap_context_t) { + .shim_data = (uint8_t *) shim_data, + .control = (shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uintptr_t) slot * + SHIM_MMAP_CONTROL_STRIDE), + .gate = (_Atomic uint32_t *) (shim_data + SHIM_MMAP_PT_GATE_OFF), + .pt_epoch = (_Atomic uint64_t *) (shim_data + SHIM_MMAP_PT_EPOCH_OFF), + .slot = slot, + }; +} + +static inline bool el1_stats_enabled(const el1_mmap_context_t *context) +{ + return context->shim_data[SHIM_GLOBALS_OFF_STATS_EN] != 0; +} + +/* Each vCPU is the sole writer of its own counter slots. */ +static inline void el1_counter_increment(_Atomic uint64_t *counter) +{ + uint64_t value = atomic_load_explicit(counter, memory_order_relaxed); + atomic_store_explicit(counter, value + 1, memory_order_relaxed); +} + +static inline void el1_mmap_counter(const el1_mmap_context_t *context, + unsigned counter) +{ + if (el1_stats_enabled(context)) + el1_counter_increment(&context->control->counters[counter]); +} + +static inline void el1_attention_counter(const el1_mmap_context_t *context) +{ + if (!el1_stats_enabled(context)) + return; + _Atomic uint64_t *counter = + (_Atomic uint64_t *) (context->shim_data + SHIM_COUNTERS_OFF) + + SHIM_COUNTER_ATTN_BAIL; + el1_counter_increment(counter); +} + +static inline bool el1_attention_pending(const el1_mmap_context_t *context) +{ + return atomic_load_explicit((_Atomic uint32_t *) (context->shim_data + + SHIM_GLOBALS_OFF_ATTN), + memory_order_acquire) != 0; +} + +/* Join the host-writer handshake. Arena revoke and refill both run with the + * gate closed; announcing this per-vCPU producer prevents the host from + * invalidating a descriptor between the generation check and the publication. + * The second gate read closes the check-vs-announce race: if the host closed it + * in between, withdraw without touching a PTE. + */ +static el1_gate_result_t el1_gate_enter(el1_mmap_context_t *context) +{ + if (atomic_load_explicit(context->gate, memory_order_acquire) != 0) + return EL1_GATE_CLOSED_BEFORE_ANNOUNCE; + + atomic_store_explicit(&context->control->retire.producer_active, 1, + memory_order_release); + if (atomic_load_explicit(context->gate, memory_order_acquire) != 0) { + atomic_store_explicit(&context->control->retire.producer_active, 0, + memory_order_release); + return EL1_GATE_CLOSED_AFTER_ANNOUNCE; + } + return EL1_GATE_ENTERED; +} + +static inline void el1_gate_leave(el1_mmap_context_t *context) +{ + atomic_store_explicit(&context->control->retire.producer_active, 0, + memory_order_release); +} + +/* mmap: consume a host-prepared, per-vCPU VA arena for the exact anonymous RW + * shape used by allocators: + * + * mmap(NULL, len, PROT_READ|PROT_WRITE, + * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) + * + * Each vCPU is the sole producer of its mmap publication ring and cursor. The + * host acquire-drains all publications whenever it takes mmap_lock. + */ + +static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) +{ + shim_mmap_control_t *control = context->control; + + if (el1_attention_pending(context)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_ATTENTION); + el1_attention_counter(context); + return false; + } + + uint64_t prot = saved_gprs[2]; + uint64_t flags = saved_gprs[3]; + uint64_t length; + if (saved_gprs[0] != 0 || prot != EL1_PROT_RW || + (flags & ~EL1_MAP_NORESERVE) != (EL1_MAP_PRIVATE | EL1_MAP_ANONYMOUS) || + !el1_page_round(saved_gprs[1], &length)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_SHAPE_MISS); + return false; + } + + switch (el1_gate_enter(context)) { + case EL1_GATE_ENTERED: + break; + case EL1_GATE_CLOSED_BEFORE_ANNOUNCE: + case EL1_GATE_CLOSED_AFTER_ANNOUNCE: + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + uint32_t generation = + atomic_load_explicit(&control->generation, memory_order_acquire); + if (generation != atomic_load_explicit(&control->consumer_generation, + memory_order_relaxed)) { + /* A revocation deliberately leaves the consumer generation stale. Ack + * only after the acquire load above, then make this syscall take HVC; + * the host will either leave the control disabled or publish a fresh + * arena. + */ + atomic_store_explicit(&control->consumer_generation, generation, + memory_order_relaxed); + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_GENERATION_STALE); + return false; + } + + if (!(atomic_load_explicit(&control->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + /* Reserve publication capacity before mutating the bump cursor. */ + uint32_t head = atomic_load_explicit(&control->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&control->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MMAP_RING_SIZE) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_RING_FULL); + return false; + } + + uint32_t credits = + atomic_load_explicit(&control->metadata_credits, memory_order_relaxed); + if (credits == 0) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + uint64_t address = + atomic_load_explicit(&control->cursor, memory_order_relaxed); + uint64_t cursor; + if (length >= EL1_ALIGN_THRESHOLD) { + if (el1_add_overflow(address, EL1_BLOCK_SIZE - 1, &address)) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + address &= ~(EL1_BLOCK_SIZE - 1); + } + if (el1_add_overflow(address, length, &cursor) || + cursor > + atomic_load_explicit(&control->arena_limit, memory_order_relaxed)) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + shim_mmap_entry_t *entry = &control->ring[tail & (SHIM_MMAP_RING_SIZE - 1)]; + entry->addr = address; + entry->len = length; + entry->prot = prot; + atomic_store_explicit(&control->metadata_credits, credits - 1, + memory_order_relaxed); + + /* Publish the mmap entry before exposing its range to the acquire load in + * el1_munmap_match_arena. Retirement then carries this publication to the + * host's acquire snapshot of the retire tail. + */ + atomic_store_explicit(&control->tail, tail + 1, memory_order_release); + atomic_store_explicit(&control->cursor, cursor, memory_order_release); + atomic_store_explicit(&control->pending_work, 1, memory_order_release); + + el1_gate_leave(context); + saved_gprs[0] = address; + el1_mmap_counter(context, SHIM_MMAP_COUNTER_HIT); + return true; +} + +/* munmap: invalidate an anonymous arena range synchronously at EL1, then + * release-publish a metadata retirement to this vCPU's SPSC ring. The host + * consumes it at a later natural VM exit. No shared allocator cursor is + * modified here. + * + * Validation precedes mutation, so clearing cannot discover a need to split + * after changing an earlier descriptor. A range within one L3 table retains the + * validated table pointer; other shapes use a second walk. Whole L2 blocks and + * L3 leaves are the only descriptors this path writes. + */ + +/* generation is re-read after bounds so a descriptor publication cannot be + * observed torn. + */ +static bool el1_munmap_match_arena(shim_mmap_control_t *candidate, + uint64_t start, + uint64_t end, + unsigned slot, + uint32_t *generation_out, + uint32_t *flags_out, + unsigned *slot_out) +{ + uint32_t flags = + atomic_load_explicit(&candidate->flags, memory_order_acquire); + if (!(flags & SHIM_MMAP_CTRL_ENABLED)) + return false; + uint32_t generation = + atomic_load_explicit(&candidate->generation, memory_order_acquire); + if (start < + atomic_load_explicit(&candidate->arena_base, memory_order_relaxed)) + return false; + if (end > atomic_load_explicit(&candidate->cursor, memory_order_acquire)) + return false; + if (generation != + atomic_load_explicit(&candidate->generation, memory_order_acquire)) + return false; + *generation_out = generation; + *flags_out = flags; + *slot_out = slot; + return true; +} + +/* Locate the one arena generation containing the whole range. Owner-local + * munmap normally stops at context->control; cross-vCPU retirement scans the + * remaining descriptors. + */ +static shim_mmap_control_t *el1_munmap_find_arena( + const el1_mmap_context_t *context, + uint64_t start, + uint64_t end, + uint32_t *generation_out, + uint32_t *flags_out, + unsigned *slot_out) +{ + if (el1_munmap_match_arena(context->control, start, end, context->slot, + generation_out, flags_out, slot_out)) + return context->control; + + for (unsigned slot = 0; slot < EL1_ARENA_SLOTS; slot++) { + if (slot == context->slot) + continue; + shim_mmap_control_t *candidate = + (shim_mmap_control_t *) (context->shim_data + + SHIM_MMAP_CONTROL_BASE + + (uintptr_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + if (el1_munmap_match_arena(candidate, start, end, slot, generation_out, + flags_out, slot_out)) + return candidate; + } + return NULL; +} + +static inline uint64_t el1_vale_operand(uint64_t address, unsigned level) +{ + uint64_t base = (address >> 12) & ((1ULL << 44) - 1); + if (level == 0) + return base; + /* TTL[3:2]=01 selects the 4 KiB granule; TTL[1:0] selects the level. */ + return base | ((uint64_t) (4u | level) << 44); +} + +static inline uint64_t el1_rvale_operand(uint64_t address, + uint64_t units, + unsigned scale, + unsigned level) +{ + uint64_t base = (address >> 12) & ((1ULL << 37) - 1); + return base | ((uint64_t) level << 37) | ((units - 1) << 39) | + ((uint64_t) scale << 44) | (1ULL << 46); +} + +/* Encode SCALE=0..3 RVALE1IS batches directly from the changed envelope. Each + * instruction covers up to 32 scale units: 64 pages at SCALE=0, 2048 at + * SCALE=1, 65536 at SCALE=2, and 2097152 (8 GiB) at SCALE=3. Widening to the + * scale-unit boundary only invalidates neighboring TLB entries; it cannot + * change mappings. Ranges larger than one instruction are emitted as adjacent + * batches while retaining a single completion DSB. + */ +static void el1_tlbi_pages(uint64_t start, + uint64_t end, + bool range_supported, + unsigned level) +{ + uint64_t pages = (end - start) / EL1_PAGE_SIZE; + if (pages <= 1 || (!range_supported && pages <= 8)) { + /* The Armv8.4+ range flag also establishes FEAT_TTL. */ + unsigned ttl_level = range_supported ? level : 0; + uint64_t operand = el1_vale_operand(start, ttl_level); + for (uint64_t page = 0; page < pages; page++) + el1_tlbi_vale1is(operand + page); + return; + } + + if (!range_supported) { + el1_tlbi_all(); + return; + } + + unsigned scale; + unsigned unit_shift; + if (pages <= 64) { + scale = 0; + unit_shift = 13; /* 2 pages = 8 KiB */ + } else if (pages <= 0x800) { + scale = 1; + unit_shift = 18; /* 64 pages = 256 KiB */ + } else if (pages <= 0x10000) { + scale = 2; + unit_shift = 23; /* 2048 pages = 8 MiB */ + } else { + scale = 3; + unit_shift = 28; /* 65536 pages = 256 MiB */ + } + + uint64_t unit = 1ULL << unit_shift; + uint64_t mask = unit - 1; + uint64_t first = start & ~mask; + uint64_t last; + if (el1_add_overflow(end, mask, &last)) { + el1_tlbi_all(); + return; + } + last &= ~mask; + while (first < last) { + uint64_t units = (last - first) >> unit_shift; + if (units > 32) + units = 32; + el1_tlbi_rvale1is(el1_rvale_operand(first, units, scale, level)); + first += units << unit_shift; + } +} + +typedef enum { + EL1_VALIDATE_GENERIC, /* walk per block, mixed L2/L3 shapes */ + EL1_VALIDATE_L2_BLOCKS, /* proven L2-block-only under one L0 slot */ + EL1_VALIDATE_BAD, /* corrupt or unsupported descriptor shape */ +} el1_validate_t; + +typedef struct { + _Atomic uint64_t *single_l3; + unsigned single_l3_first; + unsigned single_l3_limit; + unsigned tlbi_level; +} el1_munmap_pt_plan_t; + +/* Common large-anonymous case: the materialized envelope is 2 MiB aligned and + * consists only of L2 block descriptors (or holes retired by a sibling). Walk + * L0 once, then L1 once per 1 GiB window and scan the L2 entries linearly. The + * generic path remains the fallback for any L3 table or unusual upper shape. + */ +static el1_validate_t el1_validate_l2_blocks(uint64_t start, uint64_t end) +{ + if (((start | end) & (EL1_BLOCK_SIZE - 1)) != 0) + return EL1_VALIDATE_GENERIC; + if ((((end - 1) ^ start) >> 39) != 0) /* must stay in one L0 slot */ + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = + atomic_load_explicit(&l0[(start >> 39) & 0x1ff], memory_order_acquire); + uint64_t type = el1_desc_type(l0_desc); + if (type == 0) + return EL1_VALIDATE_L2_BLOCKS; + if (type != EL1_DESC_TABLE) + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t address = start; + while (address < end) { + uint64_t window_end = + el1_min(((address >> 30) + 1) << 30, end); /* next 1 GiB */ + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + uint64_t l1_type = el1_desc_type(l1_desc); + if (l1_type == 0) { + address = window_end; + continue; + } + if (l1_type != EL1_DESC_TABLE) + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l2 = el1_table(l1_desc); + unsigned index = (unsigned) ((address >> 21) & 0x1ff); + while (address < window_end) { + uint64_t l2_desc = + atomic_load_explicit(&l2[index], memory_order_relaxed); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type != 0) { + if (l2_type != EL1_DESC_BLOCK) /* L2 block, never L3 table */ + return EL1_VALIDATE_GENERIC; + } + index++; + address += EL1_BLOCK_SIZE; + } + } + return EL1_VALIDATE_L2_BLOCKS; +} + +/* Validation records a single-L3 clear range and the known leaf level. The + * caller charges the materialized-envelope intersection as a conservative upper + * bound rather than reading every L3 leaf merely to count exact bytes. + */ +static el1_validate_t el1_validate_generic(uint64_t request_start, + uint64_t request_end, + uint64_t start, + uint64_t end, + el1_munmap_pt_plan_t *plan) +{ + bool saw_l2 = false; + bool saw_l3 = false; + bool single_block = el1_block_start(start) == el1_block_start(end - 1); + uint64_t address = start; + while (address < end) { + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = atomic_load_explicit(&l0[(address >> 39) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l0_desc) == 0) { + address = el1_next_block(address); + continue; + } + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) + return EL1_VALIDATE_BAD; + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l1_desc) == 0) { + address = el1_next_block(address); + continue; + } + if (el1_desc_type(l1_desc) != EL1_DESC_TABLE) + return EL1_VALIDATE_BAD; + + _Atomic uint64_t *l2 = el1_table(l1_desc); + uint64_t l2_desc = atomic_load_explicit(&l2[(address >> 21) & 0x1ff], + memory_order_acquire); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type == 0) { + address = el1_next_block(address); + continue; + } + if (l2_type == EL1_DESC_TABLE) { + saw_l3 = true; + uint64_t block_end = el1_min(el1_next_block(address), end); + if (single_block) { + plan->single_l3 = el1_table(l2_desc); + plan->single_l3_first = (unsigned) ((address >> 12) & 0x1ff); + plan->single_l3_limit = + plan->single_l3_first + + (unsigned) ((block_end - address) / EL1_PAGE_SIZE); + } + address = block_end; + continue; + } + if (l2_type != EL1_DESC_BLOCK) + return EL1_VALIDATE_BAD; + + saw_l2 = true; + /* An L2 block can only be cleared as a whole. */ + uint64_t block_start = el1_block_start(address); + if (request_start > block_start || + request_end < block_start + EL1_BLOCK_SIZE) + return EL1_VALIDATE_BAD; + address = block_start + EL1_BLOCK_SIZE; + } + if (saw_l2 != saw_l3) + plan->tlbi_level = saw_l2 ? EL1_TLBI_LEVEL_L2 : EL1_TLBI_LEVEL_L3; + return EL1_VALIDATE_GENERIC; +} + +/* Mutation pass. Host writers are gated; sibling fast munmaps can only change + * the same descriptors to zero, so repeated invalidation is safe. + */ +static inline void el1_clear_l3_batch(_Atomic uint64_t *entry) +{ + __asm__ volatile( + "stp xzr, xzr, [%0, #0]\n\t" + "stp xzr, xzr, [%0, #16]\n\t" + "stp xzr, xzr, [%0, #32]\n\t" + "stp xzr, xzr, [%0, #48]" + : + : "r"(entry) + : "memory"); +} + +static inline void el1_clear_l3_pair(_Atomic uint64_t *entry) +{ + __asm__ volatile("stp xzr, xzr, [%0]" : : "r"(entry) : "memory"); +} + +static void el1_clear_l3_entries(_Atomic uint64_t *l3, + unsigned first, + unsigned limit) +{ + /* Page tables are page-aligned. Peel to a 64-byte group with scalar and + * paired stores, clear full groups, then clear paired and scalar tails. The + * caller's DSB ISHST orders every descriptor write before TLBI. + */ + while (first < limit && (first & (EL1_L3_PTES_PER_BATCH - 1)) != 0) { + if (((first & 1u) == 0) && (limit - first >= 2)) { + el1_clear_l3_pair(&l3[first]); + first += 2; + } else { + atomic_store_explicit(&l3[first], 0, memory_order_relaxed); + first++; + } + } + while (limit - first >= EL1_L3_PTES_PER_BATCH) { + el1_clear_l3_batch(&l3[first]); + first += EL1_L3_PTES_PER_BATCH; + } + while (limit - first >= 2) { + el1_clear_l3_pair(&l3[first]); + first += 2; + } + if (first < limit) + atomic_store_explicit(&l3[first], 0, memory_order_relaxed); +} + +static bool el1_clear_generic(uint64_t start, uint64_t end) +{ + uint64_t address = start; + while (address < end) { + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = atomic_load_explicit(&l0[(address >> 39) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) { + address = el1_next_block(address); + continue; + } + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l1_desc) != EL1_DESC_TABLE) { + address = el1_next_block(address); + continue; + } + + _Atomic uint64_t *l2 = el1_table(l1_desc); + _Atomic uint64_t *l2_entry = &l2[(address >> 21) & 0x1ff]; + uint64_t l2_desc = atomic_load_explicit(l2_entry, memory_order_acquire); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type == 0) { + address = el1_next_block(address); + continue; + } + if (l2_type == EL1_DESC_BLOCK) { + atomic_store_explicit(l2_entry, 0, memory_order_release); + address = el1_next_block(address); + continue; + } + if (l2_type != EL1_DESC_TABLE) + return false; /* validation invariant */ + + _Atomic uint64_t *l3 = el1_table(l2_desc); + uint64_t block_end = el1_min(el1_next_block(address), end); + unsigned first = (unsigned) ((address >> 12) & 0x1ff); + unsigned limit = + first + (unsigned) ((block_end - address) / EL1_PAGE_SIZE); + el1_clear_l3_entries(l3, first, limit); + address = block_end; + } + return true; +} + +/* Validation proved one L0 table and L2-block-only leaves. Re-walk L1 once per + * 1 GiB window, then use ordinary aligned 64-bit stores. Those stores are + * atomic; the DSB ISHST below supplies the required publication order, so a + * release store on every descriptor is unnecessary. + */ +static void el1_clear_l2_blocks(uint64_t start, uint64_t end) +{ + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = + atomic_load_explicit(&l0[(start >> 39) & 0x1ff], memory_order_relaxed); + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) + return; + _Atomic uint64_t *l1 = el1_table(l0_desc); + + uint64_t address = start; + while (address < end) { + uint64_t window_end = el1_min(((address >> 30) + 1) << 30, end); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_relaxed); + if (el1_desc_type(l1_desc) == 0) { + address = window_end; + continue; + } + + _Atomic uint64_t *l2 = el1_table(l1_desc); + unsigned index = (unsigned) ((address >> 21) & 0x1ff); + while (address < window_end) { + /* Validation plus the closed host gate proves this slot is a block + * or already zero. A sibling producer can only move it toward zero, + * so an unconditional aligned store is safe and avoids a second + * load/branch. + */ + atomic_store_explicit(&l2[index], 0, memory_order_relaxed); + index++; + address += EL1_BLOCK_SIZE; + } + } +} + +static bool el1_munmap(el1_mmap_context_t *context, uint64_t *saved_gprs) +{ + shim_mmap_control_t *control = context->control; + munmap_retire_ring_t *retire = &control->retire; + + if (el1_attention_pending(context)) { + el1_attention_counter(context); + return false; + } + + uint64_t start = saved_gprs[0]; + uint64_t length = saved_gprs[1]; + uint64_t end; + if (length == 0 || (start & (EL1_PAGE_SIZE - 1)) || + (length & (EL1_PAGE_SIZE - 1)) || el1_add_overflow(start, length, &end)) + return false; + + uint32_t head = atomic_load_explicit(&retire->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&retire->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MUNMAP_RETIRE_RING_SIZE - 1) { + atomic_fetch_add_explicit(&control->munmap_retire_near_full, 1, + memory_order_relaxed); + return false; /* near-full => batched HVC drain */ + } + uint64_t produced = + atomic_load_explicit(&retire->produced_bytes, memory_order_relaxed); + + switch (el1_gate_enter(context)) { + case EL1_GATE_ENTERED: + break; + case EL1_GATE_CLOSED_BEFORE_ANNOUNCE: + case EL1_GATE_CLOSED_AFTER_ANNOUNCE: + return false; + } + + uint32_t generation; + uint32_t arena_flags; + unsigned arena_slot; + shim_mmap_control_t *owner = el1_munmap_find_arena( + context, start, end, &generation, &arena_flags, &arena_slot); + if (!owner) { + el1_gate_leave(context); + return false; + } + + /* Reserve and fill, but do not advance tail until PTE+TLBI completion. */ + munmap_retire_entry_t *entry = + &retire->entries[tail & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + entry->addr = start; + entry->length = length; + entry->arena_generation = generation; + entry->flags = arena_slot; + + /* Refill invalidates every stale descriptor before publishing a new + * generation. Until the host records a lazy materialization in that + * generation, the entire arena is known PTE-empty and no walk/TLBI is + * needed. The host PT gate makes the generation marker stable here. + */ + uint64_t charged_pages = 0; + uint64_t walk_start = 0; + uint64_t walk_end = 0; + bool walked = false; + if (atomic_load_explicit(&owner->materialized_generation, + memory_order_acquire) == generation) { + uint64_t materialized_start = atomic_load_explicit( + &owner->materialized_start, memory_order_relaxed); + uint64_t materialized_end = atomic_load_explicit( + &owner->materialized_end, memory_order_relaxed); + walk_start = start > materialized_start ? start : materialized_start; + walk_end = el1_min(end, materialized_end); + walked = walk_start < walk_end; + } + + bool l2_blocks = false; + el1_munmap_pt_plan_t plan = {0}; + if (walked) { + el1_validate_t verdict = el1_validate_l2_blocks(walk_start, walk_end); + if (verdict == EL1_VALIDATE_L2_BLOCKS) { + l2_blocks = true; + plan.tlbi_level = EL1_TLBI_LEVEL_L2; + } else { + verdict = + el1_validate_generic(start, end, walk_start, walk_end, &plan); + if (verdict == EL1_VALIDATE_BAD) { + el1_gate_leave(context); + return false; + } + } + + /* Charge the materialized-envelope intersection rather than walking + * every L3 leaf to count exact PTE coverage. Holes can only make this + * an overestimate, which is safe for the advisory cleanup threshold; + * the host consumes the same bound from this retirement entry. + */ + charged_pages = (walk_end - walk_start) / EL1_PAGE_SIZE; + entry->flags = + (uint32_t) (charged_pages << SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT) | + (uint32_t) arena_slot; + } + + if (walked && charged_pages != 0) { + uint64_t consumed = + atomic_load_explicit(&retire->consumed_bytes, memory_order_acquire); + uint64_t charge = charged_pages << 12; + uint64_t pending; + if (el1_add_overflow(produced - consumed, charge, &pending) || + el1_add_overflow(produced, charge, &produced)) { + el1_gate_leave(context); + return false; + } + + /* Crossing the soft threshold records cleanup_requested but never + * forces this producer to HVC; another natural exit consumes the ring. + */ + if (pending > SHIM_MUNMAP_RETIRE_BYTES_SOFT) + atomic_store_explicit(&retire->cleanup_requested, 1, + memory_order_release); + + if (l2_blocks) { + el1_clear_l2_blocks(walk_start, walk_end); + } else if (plan.single_l3) { + el1_clear_l3_entries(plan.single_l3, plan.single_l3_first, + plan.single_l3_limit); + } else if (!el1_clear_generic(walk_start, walk_end)) { + el1_gate_leave(context); + return false; + } + + /* Descriptor stores -> leaf invalidation -> completed visibility before + * the retire tail release makes metadata cleanup eligible. + */ + el1_dsb_ishst(); + el1_tlbi_pages(walk_start, walk_end, + (arena_flags & SHIM_MMAP_CTRL_TLBIRANGE) != 0, + plan.tlbi_level); + el1_dsb_ish(); + el1_isb(); + + /* Release pairs with the acquire in gva_translate_perm: a host thread + * that reads the new epoch walks the cleared descriptors instead of + * serving its cached translation. pt_gen never sees this path. + */ + atomic_fetch_add_explicit(context->pt_epoch, 1, memory_order_release); + } + + atomic_store_explicit(&retire->produced_bytes, produced, + memory_order_relaxed); + atomic_store_explicit(&retire->tail, tail + 1, memory_order_release); + atomic_store_explicit(&context->control->pending_work, 1, + memory_order_release); + el1_gate_leave(context); + saved_gprs[0] = 0; + return true; +} + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]) +{ + el1_mmap_context_t context = el1_context(saved_gprs); + switch (saved_gprs[8]) { + case EL1_SYS_MMAP: + return el1_mmap(&context, saved_gprs); + case EL1_SYS_MUNMAP: + return el1_munmap(&context, saved_gprs); + default: + return false; + } +} diff --git a/src/core/shim-mmap.h b/src/core/shim-mmap.h new file mode 100644 index 00000000..36b5850e --- /dev/null +++ b/src/core/shim-mmap.h @@ -0,0 +1,20 @@ +/* + * Freestanding EL1 mmap-family fast paths. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The assembly exception shim owns the saved-register frame and calls this + * module only for mmap-family syscalls. A true return means X0 in the saved + * frame contains the completed syscall result; false asks the shim to forward + * the original frame to HVC #5 unchanged. + */ + +#pragma once + +#include +#include + +#define EL1_SAVED_GPRS 31u + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]); diff --git a/src/core/shim.S b/src/core/shim.S index b473a2da..b5342a4f 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -30,7 +30,6 @@ * tail preserves X0/X1/X2/X30 so signal_deliver's * register writes survive. * #12 System instr trap cache maintenance logging (DC CVAU, IC IVAU, ...) - * #13 Ptrace interrupt GPRs restored from HVC #5's saved SVC frame * * X8 on return from #5, the post-syscall request: * 0 no flush @@ -229,7 +228,7 @@ .equ CB_GETRANDOM_HIT, 80 .equ CB_PGSID_HIT, 88 .equ CB_FUTEX_EAGAIN_HIT, 96 -.equ CB_FUTEX_EFAULT_HIT, 104 +.equ CB_FUTEX_FAULT_BAIL, 104 .equ CB_FUTEX_SHAPE_BAIL, 112 .equ CB_FUTEX_MATCH_BAIL, 120 .equ CB_FUTEX_WAKE_HIT, 128 @@ -456,8 +455,24 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #215 /* SYS_munmap? */ + b.eq mmap_family_fast + cmp x10, #222 /* SYS_mmap? */ + b.eq mmap_family_fast b handle_svc_0 +/* mmap and munmap share a freestanding C consumer. The saved frame remains + * authoritative: false leaves it untouched for HVC #5; true places the + * completed result in saved X0. The normal restore tail reloads every other + * guest register, so the C ABI's caller-clobbered set is private to EL1. + */ +mmap_family_fast: + mov x0, sp + bl _el1_mmap_fastpath + cbz w0, handle_svc_0 + ldr x0, [sp, #0] + b svc_restore_eret + identity_class_fast: mrs x12, tpidr_el1 /* shim-globals base */ ldar w13, [x12] /* attention flag, acquire */ @@ -485,7 +500,7 @@ getpgid_fast: mrs x12, tpidr_el1 ldar w13, [x12] cbnz w13, attn_bail - ldr x0, [x12, #0x1158] /* SHIM_IDENTITY_OFF_PGID */ + ldr x0, [x12, #0x1178] /* SHIM_IDENTITY_OFF_PGID */ COUNTER_INC CB_PGSID_HIT b svc_restore_eret @@ -495,15 +510,14 @@ getsid_fast: mrs x12, tpidr_el1 ldar w13, [x12] cbnz w13, attn_bail - ldr x0, [x12, #0x1160] /* SHIM_IDENTITY_OFF_SID */ + ldr x0, [x12, #0x1180] /* SHIM_IDENTITY_OFF_SID */ COUNTER_INC CB_PGSID_HIT b svc_restore_eret /* futex_wait_fast: serve a FUTEX_WAIT or FUTEX_WAIT_BITSET, timed or not, whose * word differs from expected. It answers EAGAIN once an EL0 read proves the - * caller will not block, and EFAULT when that read faults - * (handle_el1_data_abort_recover below). Anything that could need a waiter, and - * every other shape, bails to the host. + * caller will not block. Faulting reads and calls that could need a waiter fall + * back to the host. * * The authority this must match is futex_should_block in runtime/futex.c, not * the Linux source: reordering the checks there is what would silently make @@ -614,13 +628,11 @@ futex_uaddr_tag: * drew, in one instruction rather than four, with no page walk, no ISB, and * no window between check and access for a sibling vCPU to unmap into. A * translation or permission fault lands in handle_el1_data_abort_recover - * and returns EFAULT, matching what the host answers for an address it - * cannot resolve. + * and falls back to the host, which can fault in a lazy mapping or reject + * an inaccessible address. * * A host-side SIGBUS (the truncated MAP_SHARED overlay HOST_SIGBUS_GUARD - * covers) is not a stage-1 fault and does not arrive here. Any FUTEX_WAIT - * caller already read this word in EL0 to learn the expected value, so it - * hits that case identically. + * covers) is not a stage-1 fault and does not arrive here. * * Relaxed, not the acquire it replaces: nothing orders a later access * against this load, and the path returns EAGAIN without ever enqueueing, @@ -651,8 +663,8 @@ futex_uaddr_tag: /* Timed shape. The timespec is read through the same LDTR the word is, so * an unreadable one takes the EL0 permission fault in hardware and - * handle_el1_data_abort_recover answers EFAULT, which is what Linux answers - * for it and in the same order. + * handle_el1_data_abort_recover forwards it to the host for lazy fault-in + * and timeout validation before the word comparison. * * Validity is deliberately narrower than the host's. futex.c accepts tv_sec * up to INT64_MAX/4; this accepts up to 2^31, about 68 years, and hands @@ -1042,7 +1054,7 @@ futex_wake_shape: dmb ish add x16, x12, #0x1000 - add x16, x16, #0x168 /* SHIM_FUTEX_WAITERS_OFF, 0x1168 */ + add x16, x16, #0x188 /* SHIM_FUTEX_WAITERS_OFF, 0x1188 */ ldr w17, [x16, x15, lsl #2] cbnz w17, futex_wake_waiter_bail @@ -1060,7 +1072,7 @@ futex_wake_waiter_bail: futex_shape_bail: /* Not the shape this path serves: another futex command, a zero bitset, a * non-NULL timeout, or an unaligned or tagged uaddr. An unreadable address - * is not here: LDTR faults on it and the recovery tail answers EFAULT. One + * is not here: LDTR faults on it and the recovery tail forwards it. One * counter rather than a taxonomy; splitting it further would only subdivide * the answer to "how often is the servable shape seen". */ @@ -1255,7 +1267,7 @@ not_svc: /* EC=0x25: the urandom store or futex LDTR can fault after a sibling vCPU * revokes the page. * - * Return EFAULT rather than halting the VM. + * Recover through the access-specific tail rather than halting the VM. */ cmp x10, #0x25 b.eq handle_el1_data_abort_recover @@ -1269,7 +1281,7 @@ not_svc: b handle_el0_fault /* EL0 (M=0): forward for signal delivery */ /* handle_el1_data_abort_recover: tag a data abort whose faulting PC sits inside - * the futex ldtr or the urandom-copy strb region as a recoverable EFAULT. + * the futex ldtr or the urandom-copy strb region as a recoverable fault. * * Layout invariants exploited here: * - SAVE_GPRS allocated 256 bytes for THIS (inner) entry on top of @@ -1308,19 +1320,16 @@ handle_el1_data_abort_recover: b.hs 3f 5: - /* The futex load holds no lock or exclusive monitor. Its recovery slot is - * otherwise the same shape as urandom's: discard the nested exception - * frame, restore the SVC return state, and report Linux EFAULT. This is the - * only way the futex path reports EFAULT, since LDTR takes the EL0 - * permission fault itself rather than being gated by a probe. + /* A missing PTE may belong to a valid lazy mapping. Restore the outer SVC + * state and let the host fault it in or return EFAULT. No lock or exclusive + * monitor is held across this fallback. */ add sp, sp, #256 ldp x9, x10, [sp], #16 msr elr_el1, x9 msr spsr_el1, x10 - mov x0, #-14 - COUNTER_INC CB_FUTEX_EFAULT_HIT - b svc_restore_eret + COUNTER_INC CB_FUTEX_FAULT_BAIL + b handle_svc_0 3: adr x12, urandom_strb_1byte_start @@ -1560,6 +1569,7 @@ handle_el0_fault: .Lel0_fault_tlbi_full: /* Broadcast TLB + conditional I-cache flush. X11=0 skips IC IALLU. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Lel0_fault_full_no_ic @@ -1581,6 +1591,7 @@ handle_el0_fault: mov x13, x11 ubfx x11, x9, #12, #44 mov x12, x10 + dsb ishst 4: tlbi vae1is, x11 add x11, x11, #1 subs x12, x12, #1 @@ -1595,7 +1606,10 @@ handle_el0_fault: .Lel0_fault_tlbi_rvae: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE). X9 carries the pre-encoded - * operand (baddr | NUM<<39 | TG=01<<46); X11 the I-cache hint. */ + * operand (baddr | NUM<<39 | SCALE<<44 | TG=01<<46); X11 the I-cache + * hint. + */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Lel0_fault_rvae_no_ic @@ -1639,6 +1653,7 @@ tlbi_restore_eret: * into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x0, x0, #12, #44 + dsb ishst tlbi vae1is, x0 dsb ish ic iallu @@ -1712,6 +1727,7 @@ handle_svc_0: * 1 = broadcast TLBI VMALLE1IS * 2 = execve replaced register state (drop frame + flush) * 3 = selective TLBI VAE1IS over X10 pages starting at X9 + * 4 = single-shot TLBI RVAE1IS with encoded operand in X9 * 5. Resume vCPU (execution continues below) */ hvc #5 @@ -1740,6 +1756,7 @@ tlbi_full: * the shim must IC IALLU; zero means a data-only PT change and the I-cache * invalidation is skipped. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Ltlbi_full_skip_ic @@ -1774,6 +1791,7 @@ tlbi_selective: */ ubfx x11, x9, #12, #44 /* x11 = VA[55:12] (current page operand) */ mov x12, x10 /* x12 = remaining page counter */ + dsb ishst 3: tlbi vae1is, x11 add x11, x11, #1 /* next page (operand is in 4 KiB units) */ subs x12, x12, #1 @@ -1789,11 +1807,11 @@ tlbi_selective: tlbi_range_large: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE, ARMv8.4+). The host has encoded * the full operand in X9: baddr (VA >> 12), TTL=0, NUM in bits [43:39], - * SCALE=0, ASID=0. One instruction covers up to 64 pages, avoiding the - * broadcast TLBI VMALLE1IS that the prior selective cap forced for - * 17..64-page ranges. X11 carries the I-cache hint as in tlbi_full / + * SCALE in bits [45:44], ASID=0. One instruction covers up to + * TLBI_RVAE_MAX_PAGES. X11 carries the I-cache hint as in tlbi_full / * tlbi_selective. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Ltlbi_rvae_skip_ic diff --git a/src/proved/align.h b/src/proved/align.h index 42608aea..b1eda0d6 100644 --- a/src/proved/align.h +++ b/src/proved/align.h @@ -60,7 +60,7 @@ ensures binary: \result == 0 || \result == 1; ensures rejects_only_on_wrap: \result != 0 <==> (x % align == 0 - || (x / align + 1) * align <= UINT64_MAX); + || x / align < UINT64_MAX / align); ensures aligned: \result != 0 ==> (\exists integer k; *out == k * align); ensures never_below: \result != 0 ==> *out >= x; diff --git a/src/proved/mmap-fastpath.h b/src/proved/mmap-fastpath.h new file mode 100644 index 00000000..88905558 --- /dev/null +++ b/src/proved/mmap-fastpath.h @@ -0,0 +1,234 @@ +/* + * EL1 mmap fast-path arena sizing and capacity arithmetic: the parts a proof + * can reach + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Each per-vCPU arena is a bump allocator: EL1 hands out [cursor, cursor+len) + * and advances cursor until a request no longer fits, at which point the host + * refills or grows the arena from the guest's own mmap history. Every value + * here is guest-influenced (request_len is the guest's mmap length, window_max + * is derived from a history of guest-chosen lengths), so a slip either wedges + * the allocator (undersizes an arena forever) or, in + * mmap_fastpath_request_fits, accepts a request that runs past arena_limit. + * + * Split out of mem.c because mem.c cannot be given to Frama-C: it includes + * sys/mman.h and the HVF headers, which the analyzer's libc does not model. + * These functions need nothing but stdint.h plus proved/align.h, so make + * verify-mmapfastpath proves this header directly. + * + * MMAP_FAST_ARENA_MIN/MAX/TARGET_ENTRIES and MMAP_FAST_PUBLICATION_WINDOW live + * here rather than in core/mmap-fastpath.h so the sizing policy has one + * definition; core/mmap-fastpath.h includes this header for them rather than + * repeating the numbers where a proof cannot see whether they still match. + */ + +#pragma once + +#include +#include + +#include "proved/align.h" + +#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) +#define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_ARENA_TARGET_ENTRIES 32u + +/* Registrations the arena sizer looks back over. Sixteen is long enough to + * outlive a burst of one allocation size (so a phase change does not resize the + * arena on its first mapping) and short enough that an outlier washes out + * within a few dozen mappings. + */ +#define MMAP_FAST_PUBLICATION_WINDOW 16u + +/* Whether a request of len bytes still fits the bump cursor before limit. + * + * Requests at least block_align wide are served block-aligned, matching the + * fast path's block-granular VA carve (guest_invalidate_ptes clears a whole + * block, so a sub-block bump allocation inside a block already carrying a live + * mapping would straddle stale and fresh PTEs). Requests below that threshold + * draw straight from cursor. + * + * len == 0 is its own case rather than falling out of the general fits test: + * mmap_fastpath_topup_locked calls this with len == 0 to ask "is there any room + * left", and an arena with cursor == limit has none, so that case needs cursor + * < limit rather than window_fits's cursor <= limit (which would call a + * completely full arena still fitting a zero-length request). + * + * Only the len >= block_align case is stated one-directionally (\result ==> + * room existed at the unaligned cursor, not the converse): its true value comes + * from align_up_ok, which bounds the aligned start (never_below, + * rounds_up_once) rather than pinning it to a closed-form expression restated + * here. That rules out a function reporting a fit that ends up past limit. + * Chasing the converse would mean re-deriving align_up_ok's own arithmetic in + * this contract instead of resting on its proof. + */ +/*@ + requires block_align > 0; + assigns \nothing; + ensures zero_len: + len == 0 ==> (\result <==> cursor < limit); + ensures small_request: + (len != 0 && len < block_align) ==> + (\result <==> (cursor <= limit && len <= limit - cursor)); + ensures large_request_sound: + (len != 0 && len >= block_align && \result) ==> + (cursor <= limit && len <= limit - cursor); + */ +static inline bool mmap_fastpath_request_fits(uint64_t cursor, + uint64_t limit, + uint64_t len, + uint64_t block_align) +{ + if (!len) + return cursor < limit; + uint64_t start = cursor; + if (len >= block_align && !align_up_ok(cursor, block_align, &start)) + return false; + return window_fits(start, len, limit); +} + +/* Ratio the doubling loop below relies on: MMAP_FAST_ARENA_MAX is exactly + * MMAP_FAST_ARENA_MIN doubled 9 times. Guards the loop's trip-count bound + * against either constant changing without updating the other. + */ +_Static_assert( + MMAP_FAST_ARENA_MAX == MMAP_FAST_ARENA_MIN * 512, + "mmap_fastpath_pow2_clamped's loop bound assumes MAX == MIN << 9"); + +/*@ + axiomatic ArenaPow2 { + logic integer arena_pow2(integer k); + axiom arena_pow2_zero: arena_pow2(0) == 1; + axiom arena_pow2_succ: + \forall integer k; k >= 0 ==> arena_pow2(k + 1) == 2 * arena_pow2(k); + } +*/ + +/* Smallest power of two in [MMAP_FAST_ARENA_MIN, MMAP_FAST_ARENA_MAX] that is + * at least value, or the nearer bound when value falls outside that range. + * + * This used to be the classic bit-smear round-up-to-power-of-two (value--; + * value |= value >> 1; ...; return value + 1;). Every claim reaching through + * that form, including a bound as simple as \result >= MMAP_FAST_ARENA_MIN, + * turned out to be a bitvector-shaped goal (OR only sets bits, so value | value + * >> k >= value for any k, but that is a property of the bit pattern, not of + * linear arithmetic), and neither alt-ergo nor z3 discharges even a single OR + * step of it here; proved/align.h's own history is the same shape one level + * down. Same semantics, different algorithm: this doubles from + * MMAP_FAST_ARENA_MIN instead, which is linear arithmetic all the way through + * and discharges completely, including the power-of-two and covering properties + * the smear form could not get past its two boundary branches. It costs at most + * 9 iterations (see the _Static_assert above) where the smear was 6 fixed + * shifts; both run once per arena refill, not per mmap, so the difference does + * not reach the hot path this feeds. + */ +/*@ + assigns \nothing; + ensures bounded_low: \result >= MMAP_FAST_ARENA_MIN; + ensures bounded_high: \result <= MMAP_FAST_ARENA_MAX; + ensures covers: + value <= MMAP_FAST_ARENA_MAX ==> \result >= value; + ensures clamp_low: + value <= MMAP_FAST_ARENA_MIN ==> \result == MMAP_FAST_ARENA_MIN; + ensures clamp_high: + value >= MMAP_FAST_ARENA_MAX ==> \result == MMAP_FAST_ARENA_MAX; + */ +static inline uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +{ + if (value <= MMAP_FAST_ARENA_MIN) + return MMAP_FAST_ARENA_MIN; + if (value >= MMAP_FAST_ARENA_MAX) + return MMAP_FAST_ARENA_MAX; + uint64_t p = MMAP_FAST_ARENA_MIN; + /*@ ghost int k = 0; */ + /*@ + loop invariant p_eq: p == MMAP_FAST_ARENA_MIN * arena_pow2(k); + loop invariant k_range: 0 <= k <= 9; + loop assigns p, k; + loop variant 9 - k; + */ + while (p < value) { + p += p; + /*@ ghost k = k + 1; */ + } + return p; +} + +/* Largest registration still inside a fixed-size history window. window must + * point at MMAP_FAST_PUBLICATION_WINDOW live entries; the caller passes + * shim_mmap_control_t's publication_window array, which this header does not + * name so it never has to see that struct's _Atomic fields. + * + * Only the upper bound is stated, not that \result is itself one of the + * entries: a ghost witness index tracking argmax alongside max proves its own + * base case and the two ensures it would support (both go through by taking the + * invariant as given), but the loop-invariant preservation step itself times + * out under both alt-ergo and z3 at 90s, ten times FRAMAC_TIMEOUT, while the + * plain upper-bound invariant below proves in milliseconds. The upper bound is + * also the only property arena_size's sizing math actually needs. + */ +/*@ + requires \valid_read(window + (0 .. MMAP_FAST_PUBLICATION_WINDOW - 1)); + assigns \nothing; + ensures upper_bound: + \forall integer i; 0 <= i < MMAP_FAST_PUBLICATION_WINDOW ==> + window[i] <= \result; + */ +static inline uint64_t mmap_fastpath_window_max( + const uint64_t window[MMAP_FAST_PUBLICATION_WINDOW]) +{ + uint64_t max = 0; + /*@ + loop invariant bound: 0 <= i <= MMAP_FAST_PUBLICATION_WINDOW; + loop invariant prefix_le_max: + \forall integer j; 0 <= j < i ==> window[j] <= max; + loop assigns i, max; + loop variant MMAP_FAST_PUBLICATION_WINDOW - i; + */ + for (unsigned i = 0; i < MMAP_FAST_PUBLICATION_WINDOW; i++) + if (window[i] > max) + max = window[i]; + return max; +} + +/* Target arena size covering both the recent registration history (window_max) + * and the request that is about to be served (request_len), clamped to + * [MMAP_FAST_ARENA_MIN, MMAP_FAST_ARENA_MAX]. + * + * The bound ensures below rest on mmap_fastpath_pow2_clamped's own + * bounded_low/bounded_high: adaptive and covering are each either + * MMAP_FAST_ARENA_MIN (the guard's false branch) or a pow2_clamped result, both + * now fully covered by that function's contract. -wp-rte separately proves this + * function's own arithmetic sound regardless: the two multiplication guards + * (window_max > MAX / target_entries, request_len > MAX / 2) mean neither + * window_max * target_entries nor request_len * 2 can overflow, and there is no + * division-by-zero. + */ +/*@ + assigns \nothing; + ensures result_ge_min: \result >= MMAP_FAST_ARENA_MIN; + ensures result_le_max: \result <= MMAP_FAST_ARENA_MAX; + */ +static inline uint64_t mmap_fastpath_arena_size(uint64_t window_max, + uint64_t request_len) +{ + uint64_t adaptive = MMAP_FAST_ARENA_MIN; + if (window_max) { + const uint64_t target_entries = MMAP_FAST_ARENA_TARGET_ENTRIES; + uint64_t target = window_max > MMAP_FAST_ARENA_MAX / target_entries + ? MMAP_FAST_ARENA_MAX + : window_max * target_entries; + adaptive = mmap_fastpath_pow2_clamped(target); + } + + uint64_t covering = MMAP_FAST_ARENA_MIN; + if (request_len) { + uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 + ? MMAP_FAST_ARENA_MAX + : request_len * 2; + covering = mmap_fastpath_pow2_clamped(target); + } + return adaptive > covering ? adaptive : covering; +} diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index 1afe03c0..f3012feb 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -191,7 +191,7 @@ int fork_ipc_recv_fds(int sock, int *fds, int max_count, int *out_count) return 0; } -int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm) +int fork_ipc_send_memory_regions(int ipc_sock, guest_t *g, bool use_shm) { if (use_shm) { uint32_t zero_regions = 0; @@ -202,9 +202,9 @@ int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm) #define MAX_USED_REGIONS 16 used_region_t used[MAX_USED_REGIONS]; unsigned int shim_sz = proc_get_shim_size(); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int nregions = guest_get_used_regions(g, shim_sz, used, MAX_USED_REGIONS); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint32_t num_regions = (uint32_t) nregions; if (fork_ipc_write_all(ipc_sock, &num_regions, sizeof(num_regions)) < 0) @@ -802,6 +802,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced) { @@ -854,6 +855,9 @@ int fork_ipc_send_process_state(int ipc_sock, fork_ipc_write_all(ipc_sock, regions_snapshot, num_guest_regions * sizeof(guest_region_t)) < 0) return -1; + if (fork_ipc_write_all(ipc_sock, dirty_blocks_snapshot, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) + return -1; if (fork_ipc_write_all(ipc_sock, &num_preannounced, sizeof(num_preannounced)) < 0) @@ -1077,6 +1081,12 @@ int fork_ipc_recv_process_state(int ipc_fd, g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); + if (fork_ipc_read_all(ipc_fd, g->dirty_blocks, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) { + log_error("fork-child: failed to read dirty block bitmap"); + return -1; + } + uint32_t num_preannounced = 0; if (fork_ipc_read_all(ipc_fd, &num_preannounced, sizeof(num_preannounced)) < 0) { diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index 5533c968..61c730a7 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C4651U /* "ELFQ" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C4652U /* "ELFR" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -130,7 +130,7 @@ int fork_ipc_read_all(int fd, void *buf, size_t len); int fork_ipc_send_fds(int sock, const int *fds, int count); int fork_ipc_recv_fds(int sock, int *fds, int max_count, int *out_count); -int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm); +int fork_ipc_send_memory_regions(int ipc_sock, guest_t *g, bool use_shm); int fork_ipc_recv_memory_regions(int ipc_fd, guest_t *g); int fork_ipc_send_fd_table(int ipc_sock); @@ -150,6 +150,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced); int fork_ipc_recv_process_state(int ipc_fd, diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index 403e6485..b495f36e 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -59,6 +59,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/fork-state.h" @@ -303,6 +304,7 @@ int fork_child_main(int ipc_fd, guest_destroy(&g); return 1; } + guest_rebuild_pte_present(&g); if (fork_ipc_recv_fd_table(ipc_fd, &g) < 0) { log_error("fork-child: failed to receive fd table"); @@ -539,6 +541,11 @@ int fork_child_main(int ipc_fd, */ shim_globals_rebuild_urandom_bitmap(); + if (!verbose) + mmap_fastpath_prepare_vcpu(&g, current_thread); + else + mmap_fastpath_disable(&g); + /* Now that current_thread is set, apply signal state. This must happen * after thread_register_main() so the per-thread blocked mask and altstack * are properly restored to the thread entry. @@ -610,7 +617,7 @@ typedef struct { vcpu_simd_state_t simd_state; } thread_create_args_t; -static void resolve_clone_stack_range(const guest_t *g, +static void resolve_clone_stack_range(guest_t *g, uint64_t child_stack, uint64_t *start_out, uint64_t *end_out) @@ -626,13 +633,10 @@ static void resolve_clone_stack_range(const guest_t *g, if (sp_off == 0 || sp_off > g->guest_size) return; - /* The region array is mutated under mmap_lock by any concurrent mmap or - * munmap, and clone does not otherwise take it. Reading it unlocked is a - * data race on g->regions and g->nregions, reported by ThreadSanitizer as - * soon as a sibling allocates while another thread clones. Neither caller - * holds a lock here, and mmap_lock is order 1, so taking it is safe. + /* The region array is mutated under mmap_lock. The acquire also drains EL1 + * mmap publications before clone resolves a newly allocated stack. */ - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); const guest_region_t *r = guest_region_find(g, sp_off - 1); if (r) { if (start_out) @@ -640,7 +644,7 @@ static void resolve_clone_stack_range(const guest_t *g, if (end_out) *end_out = r->end; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* Forward declaration: worker entry runs after sys_clone_thread */ @@ -1106,9 +1110,10 @@ static void *thread_create_and_run(void *arg) * how pthread_join works in musl: the joining thread does FUTEX_WAIT on * this address until it becomes 0. * - * Drain deferred stack munmaps before the store, not merely before the - * wake: a joiner polling the tid never reaches FUTEX_WAIT, so ordering the - * drain against the wake alone lets it reuse the VA while still mapped. + * Drain any deferred munmap before publishing clear_child_tid. A joiner may + * observe the zero without ever sleeping in FUTEX_WAIT, then reuse the + * freed VA immediately; ordering only the wake after cleanup leaves a + * window where MAP_FIXED_NOREPLACE still sees the old stack VMA. */ mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; @@ -1383,14 +1388,14 @@ static void *vm_clone_thread_run(void *arg) /* Set per-thread TLS pointer and enter worker run loop */ current_thread = t; thread_fork_barrier_check(); - log_debug("vm_clone tid=%lld starting on vCPU", (long long) thread_tid(t)); int wait_status = 0; int exit_code = vcpu_run_loop(vcpu, vexit, g, verbose, 0, &wait_status); - /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: drain before - * the store, so a joiner that never blocks cannot reuse the VA early. + /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: the zero + * itself, not just the futex wake, releases a joiner, so publish it only + * after the deferred stack mapping is gone. */ mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; @@ -1528,6 +1533,17 @@ int64_t sys_clone(hv_vcpu_t vcpu, if ((flags & ~(uint64_t) 0xff) & LINUX_CLONE3_NS_FLAGS) return -LINUX_EINVAL; + /* Once an anonymous arena allocation becomes a live thread stack, munmap + * must pass through thread_collect_and_defer_stack_ranges(). Revoke arena + * generations before publishing the stack to the thread table so no EL1 + * fast munmap can bypass that lifetime rule. + */ + if (child_stack != 0) { + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); + } + /* CLONE_THREAD: create a new thread in the same VM (not a new process) */ if (flags & LINUX_CLONE_THREAD) { return sys_clone_thread(vcpu, g, flags, child_stack, stack_map_start, @@ -1734,6 +1750,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, mmap_fork_anon_shared_txn_t *anon_shared_txn = NULL; guest_region_t *regions_snapshot = NULL; + uint64_t *dirty_blocks_snapshot = NULL; guest_region_t preannounced_snapshot[GUEST_MAX_PREANNOUNCED]; int snapshot_shm_fd = -1; bool siblings_quiesced = false; @@ -1964,6 +1981,10 @@ int64_t sys_clone(hv_vcpu_t vcpu, } memcpy(regions_snapshot, g->regions, snap_sz); } + dirty_blocks_snapshot = malloc(sizeof(g->dirty_blocks)); + if (!dirty_blocks_snapshot) + goto fail_snapshot; + memcpy(dirty_blocks_snapshot, g->dirty_blocks, sizeof(g->dirty_blocks)); int npreannounced_snapshot = g->npreannounced; if (npreannounced_snapshot > 0) { memcpy(preannounced_snapshot, g->preannounced, @@ -1988,8 +2009,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, uint32_t num_preannounced = (uint32_t) npreannounced_snapshot; if (fork_ipc_send_process_state( ipc_sock, regions_snapshot, num_guest_regions, - regions_tracker_stale_snapshot, preannounced_snapshot, - num_preannounced) < 0) { + regions_tracker_stale_snapshot, dirty_blocks_snapshot, + preannounced_snapshot, num_preannounced) < 0) { log_error("clone: failed to send process state"); goto fail_snapshot; } @@ -2070,6 +2091,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, child_host_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); return child_guest_pid; @@ -2077,6 +2099,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, fail_snapshot: proc_cancel_child(child_guest_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 5e583bda..81a457d5 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -346,6 +346,24 @@ static inline bool futex_uaddr_is_aligned(uint64_t uaddr) return (uaddr & 0x3) == 0; } +static uint32_t *futex_word_nofault(const guest_t *g, + uint64_t uaddr, + int required_perms) +{ + uint64_t avail = 0; + uint32_t *word = guest_ptr_avail_nofault(g, uaddr, &avail, required_perms); + return word && avail >= sizeof(*word) ? word : NULL; +} + +/* Keep materialized futex words off mmap_lock. A miss is resolved before any + * bucket lock is held. + */ +static void futex_prefault_word(const guest_t *g, uint64_t uaddr) +{ + if (!futex_word_nofault(g, uaddr, MEM_PERM_R)) + guest_lazy_faultin(g, uaddr, sizeof(uint32_t)); +} + /* Poll the guest itimer and pending-signal state on behalf of a bucket waiter. * Lock order forbids doing that under the bucket lock (bucket is 7, sig_lock is * 4), so this drops b->lock and retakes it. The caller must re-check @@ -440,7 +458,7 @@ static int64_t futex_should_block(const guest_t *g, uint32_t expected, uint32_t **word_out) { - uint32_t *word = (uint32_t *) guest_ptr(g, uaddr); + uint32_t *word = futex_word_nofault(g, uaddr, MEM_PERM_R); if (!word) return -LINUX_EFAULT; if (word_out) @@ -879,6 +897,7 @@ static int64_t futex_os_sync_wait(guest_t *g, return -LINUX_EINVAL; } + futex_prefault_word(g, uaddr); uint32_t *host_addr; int64_t block = futex_should_block(g, uaddr, expected, &host_addr); if (block != 0) @@ -1124,6 +1143,8 @@ static int64_t futex_wait_inner(unsigned *pub_bucket_out, return -LINUX_EAGAIN; } + futex_prefault_word(g, uaddr); + pthread_mutex_lock(&b->lock); /* Read the futex word while holding the bucket lock, so the enqueue below @@ -1472,6 +1493,9 @@ static int64_t futex_requeue(guest_t *g, if (!futex_uaddr_is_aligned(uaddr) || !futex_uaddr_is_aligned(uaddr2)) return -LINUX_EINVAL; + if (do_cmp) + futex_prefault_word(g, uaddr); + unsigned idx_src = futex_hash(uaddr); unsigned idx_dst = futex_hash(uaddr2); futex_bucket_t *b_src = &buckets[idx_src]; @@ -1657,6 +1681,8 @@ static int64_t futex_wake_op(guest_t *g, if (!futex_wake_op_supported(wake_op)) return -LINUX_ENOSYS; + futex_prefault_word(g, uaddr2); + unsigned idx1 = futex_hash(uaddr); unsigned idx2 = futex_hash(uaddr2); futex_bucket_t *b1 = &buckets[idx1]; @@ -1674,7 +1700,7 @@ static int64_t futex_wake_op(guest_t *g, } /* Atomically modify *uaddr2 */ - uint32_t *word2 = (uint32_t *) guest_ptr_w(g, uaddr2); + uint32_t *word2 = futex_word_nofault(g, uaddr2, MEM_PERM_W); if (!word2) { if (idx1 != idx2) pthread_mutex_unlock(&b2->lock); @@ -2448,6 +2474,8 @@ int64_t sys_futex_waitv(guest_t *g, */ if (!futex_uaddr_is_aligned(elts[i].uaddr)) return -LINUX_EINVAL; + + futex_prefault_word(g, elts[i].uaddr); } waitv_shared_t shared; diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index fa5b5017..ca21e888 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -1737,7 +1737,7 @@ static int proc_build_maps_entries(const guest_t *g, int result = -1; int saved_errno = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire_raw(); /* Convert regions[] to maps entries. regions[] is already sorted by start * address. The MAP_SHARED/MAP_ANONYMOUS/MAP_NORESERVE bits are preserved in @@ -1792,7 +1792,7 @@ static int proc_build_maps_entries(const guest_t *g, out_unlock: saved_errno = errno; - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release_raw(); if (result < 0) { maps_entries_destroy(&entries); errno = saved_errno; diff --git a/src/syscall/exec.c b/src/syscall/exec.c index 37e4d9f4..315dc11c 100644 --- a/src/syscall/exec.c +++ b/src/syscall/exec.c @@ -834,23 +834,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, uint64_t envp_gva, const char *host_path) { - /* Release mmap_lock, which this thread's sc_execve wrapper holds, before - * touching the slot at all. - * - * Waiting for the slot under it deadlocks the second of two concurrent - * requesters against the leader: the leader takes the first request, - * releases exec_handoff_lock, and blocks acquiring mmap_lock to run it, - * while this thread holds mmap_lock waiting for a slot only that leader can - * free. Neither side is torn down, so no stop check breaks the cycle. - * - * Safe to drop this early because the handoff is the first thing sys_execve - * does for a non-leader, so nothing has yet read what mmap_lock protects, - * and the guest addresses published below are dereferenced by the leader - * under its own mmap_lock rather than here. Re-taken on every return path - * so the wrapper's unlock stays balanced. - */ - pthread_mutex_unlock(&mmap_lock); - pthread_mutex_lock(&exec_handoff_lock); /* Wait for the slot. A concurrent handoff either fails (freeing the slot) @@ -858,7 +841,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, */ if (!exec_handoff_wait_for(HANDOFF_EMPTY)) { pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return -LINUX_EINTR; } @@ -877,7 +859,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, exec_handoff_set_state(HANDOFF_EMPTY); pthread_cond_broadcast(&exec_handoff_cond); pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return -LINUX_ENAMETOOLONG; } exec_handoff.blocked_mask = @@ -906,7 +887,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, } pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return result; } @@ -967,14 +947,8 @@ int64_t exec_run_handoff(hv_vcpu_t vcpu, guest_t *g, bool verbose) saved_mask = signal_save_blocked(); signal_set_blocked(adopt_mask); - /* sys_execve is written to run with mmap_lock held, which on the direct - * path its sc_execve wrapper takes. This path comes from the run loop, so - * take it here instead. - */ - pthread_mutex_lock(&mmap_lock); int64_t rc = sys_execve(vcpu, g, path_gva, argv_gva, envp_gva, verbose, host_path); - pthread_mutex_unlock(&mmap_lock); pthread_mutex_lock(&exec_handoff_lock); if (rc == SYSCALL_EXEC_HAPPENED) { @@ -1768,21 +1742,15 @@ int64_t sys_execve(hv_vcpu_t vcpu, * already have destroyed. It also has to precede the CLOEXEC sweep and * guest_reset: a sibling parked in read() on a fd about to close, or still * executing the old image's code, winds down against the memory and fd - * table its guest still expects. Both callers hold mmap_lock (order 1) - * across the whole syscall, and the teardown must not run under it: a - * sibling blocked in pthread_mutex_lock(&mmap_lock) inside sc_brk, sc_mmap, - * sc_munmap, sc_mprotect, or its own deferred stack unmap is reachable by - * none of the teardown wakes, so it can never reach a stop check and the - * join below would always time out. Measured before this release: four - * siblings looping on mmap/munmap took the fatal path every time. - * - * Dropping it here is safe because nothing in the teardown touches guest - * memory or the region table, and re-acquiring cannot contend: by the time - * it returns 0 no other guest thread is left to hold it. + * table its guest still expects. + */ + + /* Teardown must run without mmap_lock: a sibling blocked in an mmap-family + * syscall or its deferred stack unmap cannot reach a stop check while the + * lock is held. sys_execve acquires the lock below, immediately before the + * point of no return, after every sibling has stopped. */ - pthread_mutex_unlock(&mmap_lock); int survivors = thread_exec_de_thread(); - pthread_mutex_lock(&mmap_lock); /* The refusal above is a snapshot: a sibling could have created a CLONE_VM * child in the window between it and here. de_thread neither reaps nor @@ -1854,6 +1822,14 @@ int64_t sys_execve(hv_vcpu_t vcpu, return err; } + /* Input copying above may fault in argv/env strings from lazy anonymous + * mappings, so it must run without mmap_lock held. Serialize only after all + * recoverable validation is complete and immediately before replacing the + * guest address space. From this point every failure is fatal and both + * successful return paths release the lock explicitly. + */ + mmap_lock_acquire(g); + /* Point of no return. guest_reset() zeroes all guest memory. The old * process image is gone. All validation that can fail gracefully MUST * happen above this line. Failures below are unrecoverable; elfuse exits @@ -2013,6 +1989,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, unlink(interp.resolved); exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } @@ -2165,5 +2142,6 @@ int64_t sys_execve(hv_vcpu_t vcpu, exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } diff --git a/src/syscall/fd.c b/src/syscall/fd.c index b905da4d..4e32e5e9 100644 --- a/src/syscall/fd.c +++ b/src/syscall/fd.c @@ -240,6 +240,10 @@ int64_t sys_timerfd_settime(guest_t *g, { int64_t ret = 0; + (void) guest_lazy_faultin(g, new_value_gva, sizeof(linux_itimerspec_t)); + if (old_value_gva) + (void) guest_lazy_faultin(g, old_value_gva, sizeof(linux_itimerspec_t)); + pthread_mutex_lock(&sfd_lock); int slot = timerfd_find(fd); if (slot < 0) { @@ -248,7 +252,7 @@ int64_t sys_timerfd_settime(guest_t *g, } linux_itimerspec_t its; - if (guest_read_small(g, new_value_gva, &its, sizeof(its)) < 0) { + if (guest_read_nofault(g, new_value_gva, &its, sizeof(its)) < 0) { ret = -LINUX_EFAULT; goto unlock; } @@ -282,7 +286,7 @@ int64_t sys_timerfd_settime(guest_t *g, old.it_value_nsec = remaining % NS_PER_SEC; } } - if (guest_write_small(g, old_value_gva, &old, sizeof(old)) < 0) { + if (guest_write_nofault(g, old_value_gva, &old, sizeof(old)) < 0) { ret = -LINUX_EFAULT; goto unlock; } diff --git a/src/syscall/fs.c b/src/syscall/fs.c index b9244bd4..958731ae 100644 --- a/src/syscall/fs.c +++ b/src/syscall/fs.c @@ -2401,6 +2401,10 @@ int64_t sys_getdents64(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) if (!ds) return -LINUX_ENOTDIR; + uint64_t prefault_len = + count < DIRENT64_MAX_RECLEN ? count : DIRENT64_MAX_RECLEN; + (void) guest_lazy_faultin(g, buf_gva, prefault_len); + /* Serialize the walk against a concurrent getdents64 pinning the same * stream -- see the lock field in dir_stream_t. */ @@ -2408,7 +2412,8 @@ int64_t sys_getdents64(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) int64_t ret; - if (!guest_ptr(g, buf_gva)) { + uint64_t avail = 0; + if (!guest_ptr_avail_nofault(g, buf_gva, &avail, MEM_PERM_R)) { ret = -LINUX_EFAULT; goto out; } @@ -2624,15 +2629,13 @@ int64_t sys_getdents64(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) lde.d_reclen = (uint16_t) reclen; lde.d_type = entry_type; - /* Serialize entry into temp buffer, then copy to guest via - * guest_write() which handles 2MiB block boundary crossings. - */ memcpy(entry_buf, &lde, sizeof(lde)); memcpy(entry_buf + DIRENT64_HDR_BYTES, guest_name, name_len + 1); if (pad_start < reclen) memset(entry_buf + pad_start, 0, reclen - pad_start); - if (guest_write(g, buf_gva + guest_pos, entry_buf, reclen) < 0) { + if (guest_write_nofault(g, buf_gva + guest_pos, entry_buf, reclen) < + 0) { /* readdir has already handed this entry over, so leaving now * without putting it back resumes the next call past it: the guest * gets a listing one name short that still ends at 0, which is the diff --git a/src/syscall/fuse.c b/src/syscall/fuse.c index 3622b133..db1bf5d6 100644 --- a/src/syscall/fuse.c +++ b/src/syscall/fuse.c @@ -2474,6 +2474,10 @@ int64_t fuse_dev_read(int guest_fd, fuse_session_get_locked(session); pthread_mutex_unlock(&fuse_lock); + if (count) + (void) guest_lazy_faultin( + g, buf_gva, count < FUSE_FRAME_CAP ? count : FUSE_FRAME_CAP); + /* The state came from the descriptor; the session came from the fd number. * A sibling closing and reopening guest_fd between the two makes them * describe different objects, and the read would then take its blocking @@ -2544,7 +2548,7 @@ int64_t fuse_dev_read(int guest_fd, host_fd_ref_close(¬ify_ref); return -LINUX_EINVAL; } - if (guest_write(g, buf_gva, req->frame, frame_len) < 0) { + if (guest_write_nofault(g, buf_gva, req->frame, frame_len) < 0) { pthread_mutex_unlock(&session->lock); pthread_mutex_lock(&fuse_lock); fuse_session_put_locked(session); diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 0f9a1c6e..0b08c7ee 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -81,11 +81,10 @@ * under mmap_lock only by * exec_handoff_reset, and holds sig_lock * beneath it through - * signal_restore_blocked. A requester drops - * mmap_lock before taking it: waiting for - * the slot under mmap_lock deadlocks - * against the leader, which needs that lock - * to run the request that frees the slot + * signal_restore_blocked. sys_execve does + * not acquire mmap_lock until the point of + * no return, so a requester waiting for the + * slot never holds it * sig_lock (syscall/signal.c): signal handlers/pending/blocked * thread_lock (runtime/thread.c): thread table * sfd_lock (syscall/fd.c): special fd (never held with thread_lock) @@ -176,8 +175,27 @@ typedef int guest_fd_t; typedef int host_fd_t; /* Cross-module locks. */ -extern pthread_mutex_t mmap_lock; /* Lock order: 1, mmap/brk + page tables */ -extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ +extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ + +/* Acquire drains the per-vCPU EL1 mmap rings before inspecting region state. + * Faultable guest access may acquire this lock on a miss. Prepare buffers + * before taking lower-ranked locks, then use nofault access while holding them; + * pre-faulting does not pin the mapping against concurrent changes. + */ +void mmap_lock_acquire(guest_t *g); +void mmap_lock_release(void); +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); +bool mmap_lock_held_by_current_thread(void); + +/* Ownership-tracked mutex access without changing the EL1 gate or rings. */ +void mmap_lock_acquire_raw(void); +void mmap_lock_release_raw(void); + +/* Temporarily drop mmap_lock while retaining the host PT-gate reference, then + * reacquire without taking a second reference. Used only by lazy zeroing. + */ +void mmap_lock_drop_keep_gate(void); +void mmap_lock_reacquire_with_gate(guest_t *g); /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 896ced88..3acacbdd 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -8,6 +8,7 @@ * Guest memory syscalls: brk, mmap, munmap, mprotect, mremap, madvise, msync */ +#include #include #include #include @@ -19,23 +20,1181 @@ #include #include #include +#include #include "debug/log.h" +#include "debug/syscall-hist.h" #include "utils.h" +#include "core/mmap-fastpath.h" + #include "proved/align.h" +#include "proved/mmap-fastpath.h" + +#include "runtime/thread.h" +#include "syscall/linux-wire.h" +#include "syscall/fuse.h" +#include "syscall/internal.h" +#include "syscall/mem.h" + +/* Protects mmap/brk bump allocators and page table extension. Multiple threads + * may call mmap/brk concurrently; without this lock they could get overlapping + * allocations or corrupt page table structures. + */ +static pthread_mutex_t mmap_lock = + PTHREAD_MUTEX_INITIALIZER; /* Lock order: 1 */ + +static pthread_once_t mmap_fastpath_env_once = PTHREAD_ONCE_INIT; +static bool mmap_fastpath_env_enabled; +static _Atomic bool mmap_fastpath_forced_off; + +static uint64_t find_free_gap_inner(const guest_t *g, + uint64_t length, + uint64_t min_addr, + uint64_t max_addr, + uint64_t align); +static bool mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, + shim_mmap_control_t *c); +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len, + bool speculative); + +static void mmap_fastpath_read_env(void) +{ + const char *v = getenv("ELFUSE_MMAP_FASTPATH"); + mmap_fastpath_env_enabled = + !v || (strcmp(v, "0") != 0 && strcmp(v, "false") != 0); +} + +static bool mmap_fastpath_available(const guest_t *g) +{ + pthread_once(&mmap_fastpath_env_once, mmap_fastpath_read_env); + return mmap_fastpath_env_enabled && !g->is_rosetta && + !atomic_load_explicit(&mmap_fastpath_forced_off, + memory_order_acquire) && + !syscall_hist_enabled(); +} + +static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) +{ + if (!g || !g->host_base || slot < 0 || slot >= MAX_THREADS) + return NULL; + return (shim_mmap_control_t *) ((uint8_t *) g->host_base + + g->shim_data_base + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); +} + + +static _Atomic uint32_t *mmap_fastpath_pt_gate(const guest_t *g) +{ + if (!g || !g->host_base) + return NULL; + return (_Atomic uint32_t *) ((uint8_t *) g->host_base + g->shim_data_base + + SHIM_MMAP_PT_GATE_OFF); +} + +/* mmap_lock serializes host writers. The gate extends that exclusion to EL1 + * fast munmap without making the per-vCPU producers contend with each other: + * after publishing gate=closed, wait for each producer's private active word. + */ +static void mmap_fastpath_host_gate_close(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate) + return; + uint32_t previous = + atomic_fetch_add_explicit(gate, 1, memory_order_acq_rel); + if (previous != 0) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + while (atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0) + sched_yield(); + } +} + +/* Producers are stopped by the gate. Retain reservations for any undrained + * entries, then share the remaining region capacity among enabled arenas. + * Opening the gate publishes the credits to EL1's acquire load. + */ +static void mmap_fastpath_metadata_rebalance_locked(guest_t *g) +{ + uint32_t available = GUEST_MAX_REGIONS - g->nregions; + unsigned enabled = 0; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint32_t pending = + atomic_load_explicit(&c->tail, memory_order_relaxed) - + atomic_load_explicit(&c->head, memory_order_relaxed); + if (pending > SHIM_MMAP_RING_SIZE || pending > available) { + log_fatal("mmap fast path: metadata reservation overflow"); + abort(); + } + available -= pending; + c->metadata_reserved = pending; + atomic_store_explicit(&c->metadata_credits, 0, memory_order_relaxed); + if (atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) + enabled++; + } + for (int slot = 0; slot < MAX_THREADS && enabled; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint32_t credits = (available + enabled - 1) / enabled; + uint32_t room = SHIM_MMAP_RING_SIZE - c->metadata_reserved; + if (credits > room) + credits = room; + atomic_store_explicit(&c->metadata_credits, credits, + memory_order_relaxed); + c->metadata_reserved += credits; + available -= credits; + enabled--; + } +} + +static void mmap_fastpath_host_gate_open(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (gate) { + uint32_t count = atomic_load_explicit(gate, memory_order_relaxed); + if (count == 1) + mmap_fastpath_metadata_rebalance_locked(g); + while (count != 0 && !atomic_compare_exchange_weak_explicit( + gate, &count, count - 1, memory_order_release, + memory_order_relaxed)) { + } + + /* exec resets the entire shim-data page while holding mmap_lock, + * including this implementation-only counter. Seeing zero here is + * therefore an already-open gate, not an underflow. + */ + } +} + +static _Thread_local guest_t *mmap_lock_guest; +static _Thread_local bool mmap_lock_owned; + +/* Record a length the guest actually obtained from this arena. Only real + * registrations enter the window: a request that missed and was served by the + * generic path never became arena traffic, and sizing the next arena for it + * would grow arenas for a workload that does not use them. + */ +static void mmap_fastpath_note_registration(shim_mmap_control_t *c, + uint64_t len) +{ + if (!len) + return; + c->publication_window[c->publication_seq & + (MMAP_FAST_PUBLICATION_WINDOW - 1)] = len; + c->publication_seq++; +} + +static void mmap_fastpath_window_reset(shim_mmap_control_t *c) +{ + for (unsigned i = 0; i < MMAP_FAST_PUBLICATION_WINDOW; i++) + c->publication_window[i] = 0; + c->publication_seq = 0; +} + +static void mmap_fastpath_disable_control(shim_mmap_control_t *c) +{ + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->flags, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); + atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->next_arena_size = MMAP_FAST_ARENA_MIN; + + /* Teardown (exec, fast-path disable) ends the workload the history + * described, so the next one starts from the minimum arena. + */ + mmap_fastpath_window_reset(c); + atomic_store_explicit(&c->pending_work, 0, memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +static void mmap_fastpath_drain_publications_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&c->head, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&c->tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_RING_SIZE) { + log_fatal( + "mmap fast path: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + abort(); + } + + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t arena_limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + while (head != tail) { + const shim_mmap_entry_t *e = + &c->ring[head & (SHIM_MMAP_RING_SIZE - 1)]; + uint64_t addr = e->addr; + uint64_t len = e->len; + if ((addr & (GUEST_PAGE_SIZE - 1)) || !len || + (len & (GUEST_PAGE_SIZE - 1)) || addr < arena_base || + addr > arena_limit || len > arena_limit - addr || + e->prot != (LINUX_PROT_READ | LINUX_PROT_WRITE)) { + log_fatal( + "mmap fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx arena=0x%llx..0x%llx)", + slot, (unsigned long long) addr, (unsigned long long) len, + (unsigned long long) arena_base, + (unsigned long long) arena_limit); + abort(); + } + if (c->metadata_reserved == 0) { + log_fatal("mmap fast path: publication without reservation"); + abort(); + } + if (guest_region_add_ex(g, addr, addr + len, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) { + /* EL1 already returned this address to the guest. Continuing + * without semantic metadata would turn first touch into a false + * SIGSEGV, so fail closed on the violated provisioning + * invariant instead of silently corrupting process state. + */ + log_fatal( + "mmap fast path: region metadata exhausted while " + "draining vCPU slot %d", + slot); + abort(); + } + mmap_fastpath_note_registration(c, len); + c->metadata_reserved--; + head++; + } + atomic_store_explicit(&c->head, head, memory_order_release); + } +} + +static void munmap_retire_commit_locked(guest_t *g, + const munmap_retire_entry_t *e, + uint64_t backing_start, + uint64_t backing_end) +{ + uint64_t start = e->addr; + uint64_t end = start + e->length; + + /* Publication drain ran first, so every mapping causally preceding this + * retirement is now represented in regions[]. A non-anonymous overlay in an + * arena indicates a missing revocation and must fail closed: EL1 has + * already invalidated the PTEs, so silently retaining such metadata would + * permit a later fault path to recreate them. + */ + for (int i = guest_region_first_end_above(g, start); i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= start) + continue; + if (!(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) { + log_fatal( + "munmap retire: non-fast mapping in arena " + "[0x%llx-0x%llx)", + (unsigned long long) start, (unsigned long long) end); + abort(); + } + } + + guest_materialize_wait_range_locked(g, start, end); + + /* The PTE occupancy evidence was consumed by EL1, so use the conservative + * dirty bitmap to avoid touching huge never-materialized reservations. + * Retain the dirty bits and let a future lazy materialization zero only the + * backing that is actually reused. In particular, do not charge an + * unrelated VM exit (often the next mapping's first fault) with an eager + * memset of the retired range. guest_materialize_lazy_one() zeros dirty + * backing before publishing any new descriptor, so a future reader can + * never observe stale bytes. + * + * An earlier version of this function eagerly replaced the backing of large + * dirty runs (unmap + F_PUNCHHOLE + fresh mmap + remap) once the run's + * page-accurate materialized byte count crossed a size threshold, on the + * theory that handing the zero-fill to the host's demand-zero path beats a + * future software memset. Round-trip measurement (munmap immediately + * followed by a full re-touch of the same range, not just the munmap call + * in isolation) showed the opposite at every size tried from 16 MiB to 512 + * MiB: the replace path's own unmap/remap cost, paid synchronously while + * holding mmap_lock, exceeded the deferred memset it was meant to avoid, + * and grew faster than linearly with size. Retaining dirty bits + * unconditionally is cheaper in every case measured, so this function no + * longer special-cases large runs. + */ + + guest_region_remove(g, start, end); + if (backing_end > backing_start) + guest_retire_ptes_committed(g, backing_start, backing_end); +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + /* Clear every advisory pending hint before acquire-snapshotting the rings. + * A publication racing this drain either appears in this snapshot or leaves + * its hint set for the next opportunistic drain. + * + * Acquire-snapshot every retirement tail before consuming any mmap + * publication. This is the cross-vCPU causal ordering required for "A mmap; + * publish pointer; B munmap": the acquire observes B's retire, then + * publication drain establishes A's semantic region before removal. + */ + uint32_t retire_tails[MAX_THREADS]; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + atomic_exchange_explicit(&c->pending_work, 0, memory_order_acq_rel); + retire_tails[slot] = + atomic_load_explicit(&c->retire.tail, memory_order_acquire); + } + + mmap_fastpath_drain_publications_locked(g); + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = retire_tails[slot]; + if ((uint32_t) (tail - head) > SHIM_MUNMAP_RETIRE_RING_SIZE) { + log_fatal( + "munmap retire: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + abort(); + } + + while (head != tail) { + const munmap_retire_entry_t *e = + &producer->retire + .entries[head & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + uint32_t arena_slot = + e->flags & SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK; + uint64_t charged_pages = + (e->flags & SHIM_MUNMAP_RETIRE_F_CHARGE_MASK) >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT; + uint64_t charged_bytes = charged_pages * GUEST_PAGE_SIZE; + if (arena_slot >= MAX_THREADS || !e->length || + (e->addr & (GUEST_PAGE_SIZE - 1)) || + (e->length & (GUEST_PAGE_SIZE - 1)) || + e->addr > UINT64_MAX - e->length || charged_bytes > e->length) { + log_fatal("munmap retire: invalid entry in vCPU slot %d", slot); + abort(); + } + + shim_mmap_control_t *arena = + mmap_fastpath_control(g, (int) arena_slot); + uint32_t generation = + atomic_load_explicit(&arena->generation, memory_order_acquire); + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + uint64_t end = e->addr + e->length; + if (generation != e->arena_generation || e->addr < base || + end > cursor) { + log_fatal( + "munmap retire: stale arena generation/range " + "(producer=%d arena=%u gen=%u/%u)", + slot, arena_slot, e->arena_generation, generation); + abort(); + } + + uint64_t backing_start = 0, backing_end = 0; + if (charged_bytes != 0) { + backing_start = atomic_load_explicit(&arena->materialized_start, + memory_order_relaxed); + backing_end = atomic_load_explicit(&arena->materialized_end, + memory_order_relaxed); + if (backing_start < e->addr) + backing_start = e->addr; + if (backing_end > end) + backing_end = end; + if (backing_end <= backing_start) { + log_fatal( + "munmap retire: charged entry has no materialized " + "bounds (producer=%d arena=%u range=0x%llx..0x%llx " + "marker=0x%llx..0x%llx charged=0x%llx)", + slot, arena_slot, (unsigned long long) e->addr, + (unsigned long long) end, + (unsigned long long) atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed), + (unsigned long long) atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed), + (unsigned long long) charged_bytes); + abort(); + } + } + + munmap_retire_commit_locked(g, e, backing_start, backing_end); + uint64_t consumed = atomic_load_explicit( + &producer->retire.consumed_bytes, memory_order_relaxed); + atomic_store_explicit(&producer->retire.consumed_bytes, + consumed + charged_bytes, + memory_order_release); + head++; + } + atomic_store_explicit(&producer->retire.head, head, + memory_order_release); + + /* The PT gate is closed while draining, so no producer can race this + * acknowledgement. Ring fullness remains the only hard per-vCPU + * backpressure; byte pressure is deliberately advisory. + */ + atomic_store_explicit(&producer->retire.cleanup_requested, 0, + memory_order_release); + } + + /* A stopped owner whose whole arena retired can collapse all published + * sub-extents back into its bump cursor. Like envelope reset, this must + * wait for the complete snapshot: overlapping retire records from sibling + * producers may otherwise observe a prematurely reset arena. + */ + if (current_thread && current_thread->sp_el1_slot >= 0) + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); + + /* EL1 may publish several charged retirements before this drain. Their PTEs + * are all already invalid, so clearing an arena's materialized envelope + * after the first commit would make later entries in the same snapshot lose + * their backing bounds. Restore the PTE-empty proof only after every + * snapshotted retirement has consumed the old envelope. + */ + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *arena = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&arena->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + if (base < cursor && + guest_va_next_present_block(g, base, cursor) >= cursor) { + atomic_store_explicit(&arena->materialized_start, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_end, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_generation, 0, + memory_order_release); + } + } + + /* No pt_gen bump: EL1 advanced the PT epoch as it cleared the descriptors, + * and the commits above rewrite none. + */ +} + +/* Top up the calling thread's arena before it runs dry. + * + * Refills otherwise happen only after the EL1 side has already missed: the fast + * path exhausts its arena, bails to HVC, and the host refills on the way + * through the slow path, so the mapping that discovers the exhaustion always + * pays for it. Any host path that takes mmap_lock is already positioned to + * refill for free, having paid for the lock and the gate close. + * + * The water mark is the largest recent registration rather than a fixed + * fraction: what makes an arena useless is being unable to hold the mappings + * this workload actually makes, so an arena is spent exactly when its tail no + * longer covers one of them. That also bounds what the refill abandons -- + * relocating strands the unused tail until a later gap scan recovers it, and + * this way the strand is one typical mapping rather than a slice of an arena + * whose size nothing ties to the workload. A control with no history yet never + * triggers, which keeps startup from refilling on its first lock. + * + * Cheap enough for the lock path: an unavailable fast path or a thread with no + * slot costs one predictable branch, and a healthy arena costs three relaxed + * loads plus the window scan. Only the caller's own slot is considered; + * scanning all of them here would put a MAX_THREADS loop on every acquisition. + */ +static void mmap_fastpath_topup_locked(guest_t *g) +{ + if (!mmap_fastpath_available(g) || !current_thread || + current_thread->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor > limit) + return; + + uint64_t low_water = mmap_fastpath_window_max(c->publication_window); + if (!low_water || limit - cursor > low_water) + return; + + /* One control block describes one arena generation. Replacing it while a + * mapping from the used prefix is live strands that mapping: its later + * munmap cannot prove arena ownership and falls back to the host. This is + * especially expensive when one large mapping consumes the whole arena. The + * acquire above drained publications, so regions[] is authoritative. + */ + int first = guest_region_first_end_above(g, base); + if (first < g->nregions && g->regions[first].start < cursor) + return; + + mmap_fastpath_refill_thread_locked(g, current_thread, 0, true); +} + +bool mmap_lock_held_by_current_thread(void) +{ + return mmap_lock_owned; +} + +void mmap_lock_acquire_raw(void) +{ + assert(!mmap_lock_owned); + pthread_mutex_lock(&mmap_lock); + mmap_lock_owned = true; +} + +void mmap_lock_release_raw(void) +{ + assert(mmap_lock_owned); + mmap_lock_owned = false; + pthread_mutex_unlock(&mmap_lock); +} + +static void mmap_lock_acquire_common(guest_t *g) +{ + mmap_lock_acquire_raw(); + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_acquire(guest_t *g) +{ + mmap_lock_acquire_common(g); + mmap_fastpath_topup_locked(g); +} + +static void mmap_lock_acquire_for_fork(guest_t *g) +{ + /* Fork is about to revoke every arena. Keep the drain-before-region-read + * invariant without allocating a replacement arena that cannot survive this + * critical section. + */ + mmap_lock_acquire_common(g); +} + +void mmap_lock_release(void) +{ + mmap_fastpath_host_gate_open(mmap_lock_guest); + mmap_lock_guest = NULL; + mmap_lock_release_raw(); +} + +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) +{ + mmap_fastpath_host_gate_open(g); + mmap_lock_guest = NULL; + mmap_lock_owned = false; + pthread_cond_wait(cond, &mmap_lock); + mmap_lock_owned = true; + + /* pthread_cond_wait reacquires mmap_lock directly, so preserve the + * drain-before-region-read invariant of mmap_lock_acquire(). + */ + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_drop_keep_gate(void) +{ + /* Dirty lazy-materialization drops mmap_lock around a potentially large + * memset. Retain this thread's gate reference so EL1 cannot retire the + * invalid PTE window and let the materializer recreate it afterwards. + * Another host thread may temporarily acquire mmap_lock; the refcounted + * gate remains closed until this owner finishes the materialization. + */ + mmap_lock_guest = NULL; + mmap_lock_release_raw(); +} + +void mmap_lock_reacquire_with_gate(guest_t *g) +{ + mmap_lock_acquire_raw(); + mmap_lock_guest = g; + +#if !defined(NDEBUG) + /* A retained gate prevents retirement from waiting on this owner's claim. + */ + if (g && g->host_base) { + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + assert( + atomic_load_explicit(&c->retire.head, memory_order_relaxed) == + atomic_load_explicit(&c->retire.tail, memory_order_acquire)); + } + } +#endif /* !defined(NDEBUG) */ + + /* EL1 mmap publication does not need the PT gate and may have progressed + * during the memset, so refresh semantic metadata before resuming. + */ + mmap_fastpath_drain_locked(g); +} + +static bool mmap_fastpath_has_pending_work(const guest_t *g) +{ + if (!g || !g->host_base) + return false; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (atomic_load_explicit(&c->pending_work, memory_order_acquire)) + return true; + } + return false; +} + +void mmap_fastpath_drain_vmexit(guest_t *g, bool fork_family_pending) +{ + if (!mmap_fastpath_has_pending_work(g)) + return; + + if (fork_family_pending) + mmap_lock_acquire_for_fork(g); + else + mmap_lock_acquire(g); + mmap_lock_release(); +} + +bool mmap_fastpath_current_producer_active(const guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return false; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + return c && atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0; +} + +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start >= limit || end <= base) + continue; + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed); + uint64_t lo = start > base ? start : base; + uint64_t hi = end < limit ? end : limit; + uint32_t materialized = atomic_load_explicit( + &c->materialized_generation, memory_order_relaxed); + if (materialized == generation) { + uint64_t old_lo = atomic_load_explicit(&c->materialized_start, + memory_order_relaxed); + uint64_t old_hi = atomic_load_explicit(&c->materialized_end, + memory_order_relaxed); + if (old_lo < lo) + lo = old_lo; + if (old_hi > hi) + hi = old_hi; + } + atomic_store_explicit(&c->materialized_start, lo, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, hi, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, generation, + memory_order_release); + + /* Fast-path arenas are allocated from disjoint VA ranges. Once this + * block has updated its owner, no later vCPU control can overlap it; + * avoid another 63 control-page probes on the single-vCPU hot path. + */ + break; + } +} + +/* mmap_fastpath_request_fits, mmap_fastpath_pow2_clamped, + * mmap_fastpath_window_max, and mmap_fastpath_arena_size live in + * proved/mmap-fastpath.h (included above): they are pure arithmetic over + * scalars, unlike everything else in this file, so make verify-mmapfastpath + * proves them directly instead of leaving them reviewed-by-eye. + */ + +/* speculative: refill because the arena is nearly spent, not because a request + * failed to fit. The fits check below would otherwise abandon such a call + * immediately because a zero-length request fits any arena with a byte to + * spare. + */ +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len, + bool speculative) +{ + if (!t || t->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = mmap_fastpath_control(g, t->sp_el1_slot); + if (!c) + return; + if (!mmap_fastpath_available(g)) { + mmap_fastpath_disable_control(c); + return; + } + + /* Giant requests are deliberately slow-path-only. Do not abandon a useful + * small-request arena or poison its adaptive history. + */ + if (request_len > MMAP_FAST_ARENA_MAX) + return; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); + uint64_t arena_size = mmap_fastpath_arena_size( + mmap_fastpath_window_max(c->publication_window), request_len); + if (!speculative && (flags & SHIM_MMAP_CTRL_ENABLED) && + mmap_fastpath_request_fits(cursor, limit, request_len, BLOCK_2MIB)) { + /* A capacity miss enters HVC, whose drain can rewind a completely + * retired arena before this refill check. Retaining that undersized + * arena merely because one more request fits makes the same miss recur + * every few operations and turns mmap latency into a periodic sawtooth. + * Grow an empty arena to the adaptive target; never relocate one that + * still contains allocations served by the current generation. + */ + bool empty = cursor == arena_base; + bool target_sized = + arena_base <= limit && limit - arena_base >= arena_size; + if (!empty || target_sized) + return; + } + + /* The host gate is closed and publications are drained into regions[]. + * Release the old reservation before searching for its replacement; retain + * the publication window for adaptive sizing across generations. + */ + if (flags & SHIM_MMAP_CTRL_ENABLED) + atomic_store_explicit(&c->flags, 0, memory_order_relaxed); + + /* Prefer a real hole below the current high-water mark. Active sibling + * arenas are excluded by mmap_fastpath_skip_reserved inside the gap + * allocator. Only grow mmap_next when no recyclable hole fits. + */ + uint64_t high = g->mmap_next; + if (high > g->mmap_limit) + high = g->mmap_limit; + uint64_t base = UINT64_MAX; + bool recycled = false; + if (high > MMAP_BASE) { + base = find_free_gap_inner(g, arena_size, MMAP_BASE, high, BLOCK_2MIB); + recycled = base != UINT64_MAX; + } + + if (!recycled) { + if (g->mmap_next > UINT64_MAX - (BLOCK_2MIB - 1)) { + mmap_fastpath_disable_control(c); + return; + } + base = ALIGN_UP(g->mmap_next, BLOCK_2MIB); + if (base > g->mmap_limit || arena_size > g->mmap_limit - base) { + mmap_fastpath_disable_control(c); + return; + } + } + uint64_t new_limit = base + arena_size; + + /* Carve VA only. Clearing stale descriptors once here makes every later + * bump allocation PTE-free without putting page-table work in EL1. Fresh + * bump-tail arenas beyond mmap_end cannot contain stale descriptors. + */ + if (recycled || base < g->mmap_end) { + if (guest_invalidate_ptes(g, base, new_limit) < 0) { + mmap_fastpath_disable_control(c); + return; + } + } + if (!recycled) { + g->mmap_next = new_limit; + if (g->mmap_rw_gap_hint < new_limit) + g->mmap_rw_gap_hint = new_limit; + } + + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->next_arena_size = arena_size; + + /* The window deliberately survives the generation change: sizing the next + * arena from the traffic that filled the previous one is the whole point, + * and clearing it here would restart the history at every refill. + */ + c->refill_count++; + if (recycled) + c->recycle_count++; + if (arena_size > c->peak_arena_size) + c->peak_arena_size = arena_size; + uint32_t control_flags = SHIM_MMAP_CTRL_ENABLED; + if (g_tlbi_range_supported) + control_flags |= SHIM_MMAP_CTRL_TLBIRANGE; + atomic_store_explicit(&c->flags, control_flags, memory_order_relaxed); + + /* This vCPU is stopped in HVC (or has never run), so host may acknowledge + * the freshly published descriptor on its behalf. Revocation deliberately + * does not do this, making an in-flight stale generation bail once. + */ + atomic_store_explicit(&c->consumer_generation, generation, + memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) +{ + mmap_fastpath_refill_thread_locked(g, current_thread, request_len, false); +} + +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + mmap_fastpath_refill_thread_locked(g, current_thread, request_len, false); + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + return false; + uint64_t end = start + request_len; + + if (guest_region_add_ex( + g, start, end, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL, -1) < 0) + return false; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + mmap_fastpath_note_registration(c, request_len); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + return true; +} + +/* Service mmap publication-ring backpressure without turning it into a global + * PT-gate rendezvous. The calling vCPU is stopped in HVC, so the host may + * advance only that vCPU's existing bump cursor after draining its prior + * publications. Sibling EL1 allocators own disjoint arenas and may continue + * publishing concurrently. + * + * This path deliberately refuses every operation that could require a host + * writer transaction: a pending retirement, a closed gate, arena refill or + * generation change all fall back to mmap_lock_acquire(). The acquire snapshots + * of every retire tail preserve the usual drain-before-metadata ordering for + * causally prior munmaps. + */ +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!g || !addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + mmap_lock_acquire_raw(); + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate || atomic_load_explicit(gate, memory_order_acquire) != 0) + goto miss; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = + atomic_load_explicit(&producer->retire.tail, memory_order_acquire); + if (head != tail) + goto miss; + } + + mmap_fastpath_drain_publications_locked(g); + + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + + /* Sibling producers may still consume credits. Their host-owned total + * includes both unused credits and publications, even between EL1 stores. + */ + uint32_t available = GUEST_MAX_REGIONS - g->nregions; + for (int slot = 0; slot < MAX_THREADS; slot++) { + uint32_t reserved = mmap_fastpath_control(g, slot)->metadata_reserved; + if (reserved >= available) + goto miss; + available -= reserved; + } + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_acquire); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->consumer_generation, memory_order_relaxed) != + generation) + goto miss; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start == base) { + uint64_t target = mmap_fastpath_arena_size( + mmap_fastpath_window_max(c->publication_window), request_len); + if (base > limit || limit - base < target) + goto miss; + } + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + goto miss; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + goto miss; + uint64_t end = start + request_len; + + if (guest_region_add_ex( + g, start, end, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL, -1) < 0) + goto miss; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + mmap_fastpath_note_registration(c, request_len); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + + /* Only this producer is stopped. Refill its credits from unreserved + * capacity, charging one slot for the host allocation even if it merged. + */ + uint32_t grant = SHIM_MMAP_RING_SIZE - c->metadata_reserved; + if (grant > available - 1) + grant = available - 1; + c->metadata_reserved += grant; + uint32_t credits = + atomic_load_explicit(&c->metadata_credits, memory_order_relaxed); + atomic_store_explicit(&c->metadata_credits, credits + grant, + memory_order_release); + mmap_lock_release_raw(); + return true; + +miss: + mmap_lock_release_raw(); + return false; +} + +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length) +{ + if (!current_thread || current_thread->sp_el1_slot < 0 || !length || + addr > UINT64_MAX - length) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (addr < cursor || addr >= limit) + return; -#include "runtime/thread.h" -#include "syscall/linux-wire.h" -#include "syscall/fuse.h" -#include "syscall/internal.h" -#include "syscall/mem.h" + /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race + * this descriptor update. Release an unallocated tail for an explicit hint; + * hints in the used prefix must not revoke ownership of its holes. + */ + mmap_fastpath_disable_control(c); +} -/* Protects mmap/brk bump allocators and page table extension. Multiple threads - * may call mmap/brk concurrently; without this lock they could get overlapping - * allocations or corrupt page table structures. +/* Reuse a fully released arena in place. mmap_lock acquisition has drained this + * vCPU's publication ring, and sys_munmap has removed the last semantic region + * before calling here. The PTE occupancy index is the final guard: an arena is + * rewound only when no live metadata and no valid descriptor remain. The owner + * is stopped in HVC, so resetting its private bump cursor cannot race EL1. + * Keeping base/limit/generation unchanged avoids a host refill on the next mmap + * This matters when one 32 GiB request consumes the entire maximum-sized arena. */ -pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; /* Lock order: 1 */ +static bool mmap_fastpath_rewind_control_if_clean_locked(guest_t *g, + shim_mmap_control_t *c) +{ + if (!g || !c) + return false; + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + if (base >= limit || cursor <= base) + return false; + + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= limit) + break; + if (r->end > base) + return false; + } + if (guest_va_next_present_block(g, base, limit) < limit) + return false; + + /* The stopped owner can safely discard pending sub-extents because the + * whole arena is becoming one bump-allocatable extent again. + */ + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->recycle_count++; + return true; +} + +static void mmap_fastpath_rewind_current_if_clean_locked(guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return; + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); +} + +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) +{ + mmap_lock_acquire(g); + mmap_fastpath_refill_thread_locked(g, t, 0, false); + mmap_lock_release(); +} + +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) +{ + mmap_fastpath_drain_locked(g); + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (c) + mmap_fastpath_disable_control(c); + } + + if (!shrink_high_water) + return; + uint64_t high = MMAP_BASE; + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= MMAP_BASE && r->start < g->mmap_limit && r->end > high) + high = r->end; + } + g->mmap_next = high; + if (g->mmap_rw_gap_hint > high) + g->mmap_rw_gap_hint = high; +} + +void mmap_fastpath_disable(guest_t *g) +{ + atomic_store_explicit(&mmap_fastpath_forced_off, true, + memory_order_release); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, true); + mmap_lock_release(); +} + +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr) +{ + if (!g || !start || !length) + return; + bool advanced; + do { + advanced = false; + if (*start > max_addr || length > max_addr - *start) + return; + uint64_t end = *start + length; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_acquire) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + + /* EL1 munmap classifies ownership by arena bounds, including holes + * below cursor. Keep those holes reserved until revocation. + */ + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (base < limit && *start < limit && end > base) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + } + } while (advanced); +} /* Host kernel page size (16 KiB on Apple Silicon, typically 4 KiB on Intel * macOS). MAP_FIXED requires addr/length/offset multiples of this, so an @@ -111,6 +1270,11 @@ static int64_t sync_shared_aliases_range(guest_t *g, int backing_fd, uint64_t file_start, uint64_t file_end); +static int read_file_range_to_guest(guest_t *g, + uint64_t gpa, + int fd, + uint64_t file_off, + uint64_t len); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -686,6 +1850,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, uint64_t gap_start; if (!align_up_ok(min_addr, align, &gap_start)) return UINT64_MAX; + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* Skip the prefix of regions entirely below gap_start in O(log n). After a * successful allocation the gap hint advances near or past the existing @@ -694,6 +1859,8 @@ static uint64_t find_free_gap_inner(const guest_t *g, */ for (int i = guest_region_first_end_above(g, gap_start); i < g->nregions; i++) { + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); + /* A region can still slip below gap_start after the align_up_ok advance * below skips past a smaller adjacent region; keep the cheap guard. */ @@ -727,6 +1894,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, } /* Check trailing space after all regions */ + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); if (window_fits(gap_start, length, max_addr)) return gap_start; return UINT64_MAX; /* No suitable gap found */ @@ -888,6 +2056,33 @@ static bool high_va_replaceable_gpa_base(guest_t *g, return true; } +static int mmap_high_va_preflight_locked(guest_t *g, + uint64_t addr, + uint64_t length, + bool replace_existing, + bool is_noreplace) +{ + if (!g->is_rosetta || !length || addr > UINT64_MAX - length || + addr + length > UINT64_MAX - (BLOCK_2MIB - 1) || + guest_kbuf_user_va_overlap(addr, length)) + return -LINUX_ENOMEM; + if (!region_range_overlaps(g, addr, addr + length)) + return 0; + + /* Replacement rollback keeps a host byte snapshot, capped at 256 MiB. */ + if (length > ((size_t) 256 << 20)) + return -LINUX_ENOMEM; + if (is_noreplace) + return -LINUX_EEXIST; + if (!replace_existing || + !high_va_replaceable_gpa_base(g, addr, addr + length, NULL, NULL, + NULL) || + !region_has_capacity_after_removes( + g, &(remove_range_t) {addr, addr + length}, 1, 1)) + return -LINUX_ENOMEM; + return 0; +} + /* Over the function-size limit on purpose. * * The Rosetta high-VA mapping loop plus its rollback, which has to undo page @@ -903,18 +2098,19 @@ static int64_t sys_mmap_high_va(guest_t *g, guest_fd_t fd, uint64_t offset, bool replace_existing, - bool is_noreplace) + bool is_noreplace, + int materialized_fd) { - int64_t ret = -LINUX_ENOMEM; - - if (!g->is_rosetta) - return -LINUX_ENOMEM; + int64_t ret = mmap_high_va_preflight_locked(g, addr, length, + replace_existing, is_noreplace); + if (ret < 0) + return ret; + ret = -LINUX_ENOMEM; bool is_anon = (flags & LINUX_MAP_ANONYMOUS) != 0; host_fd_ref_t backing_ref = HOST_FD_REF_INIT; int host_backing_fd = -1; int track_backing_fd = -1; - bool close_host_backing_fd = false; /* High-water mark of VA installed by the mapping loop; reachable from the * fail label so the rollback knows what to invalidate. Must be initialized @@ -969,21 +2165,6 @@ static int64_t sys_mmap_high_va(guest_t *g, * fd) already fell through here and are unchanged. */ - /* Reject wrap before reusing addr + length anywhere below. The caller - * page-rounds length, but addr is guest-supplied and a huge length against - * a high VA can still overflow. Also reject the case where addr + length is - * too close to UINT64_MAX for ALIGN_UP to round up the 2 MiB boundary - * without wrapping to 0 (which would make va_limit smaller than va_base and - * underflow backing_span). - */ - if (length == 0 || addr > UINT64_MAX - length) - return -LINUX_ENOMEM; - if ((addr + length) > UINT64_MAX - (BLOCK_2MIB - 1)) - return -LINUX_ENOMEM; - - if (guest_kbuf_user_va_overlap(addr, length)) - return -LINUX_ENOMEM; - /* Set when this call enters the replace-an-existing-mapping branch * (region_range_overlaps + replaceable + snapshots captured). Used * everywhere the function needs to decide between the fresh-allocation path @@ -995,33 +2176,11 @@ static int64_t sys_mmap_high_va(guest_t *g, */ bool replacing_existing = false; - /* Cap the byte-snapshot allocation that populate_existing needs for - * rollback. The mapping itself can still be arbitrarily large in the - * fresh-allocation path; only the replace-an-existing branch needs the - * host-side malloc, so the cap only applies to replacement. 256 MiB is - * comfortably above realistic Rosetta dynamic-linker reservations and far - * below the multi-GiB malloc bombs a hostile guest could otherwise force. - * Reject early with -ENOMEM so the caller falls back to a smaller MAP_FIXED - * footprint rather than triggering the host OOM killer. - */ - enum { HIGH_VA_SNAPSHOT_MAX = (size_t) 256 << 20 }; - - if (region_range_overlaps(g, addr, addr + length) && - length > HIGH_VA_SNAPSHOT_MAX) - return -LINUX_ENOMEM; - if (region_range_overlaps(g, addr, addr + length)) { - if (is_noreplace) - return -LINUX_EEXIST; - if (!replace_existing) - return -LINUX_ENOMEM; if (!high_va_replaceable_gpa_base(g, addr, addr + length, &replaced_gpa_base, &replaced_flags, &replaced_offset)) return -LINUX_ENOMEM; - if (!region_has_capacity_after_removes( - g, &(remove_range_t) {addr, addr + length}, 1, 1)) - return -LINUX_ENOMEM; if (guest_region_remove_prepare(g, addr, addr + length, &replaced_remove_fd) < 0) return -LINUX_ENOMEM; @@ -1065,25 +2224,8 @@ static int64_t sys_mmap_high_va(guest_t *g, } if (!is_anon) { - if (fuse_fd_refuse_mmap(fd)) { - char materialized_path[PATH_MAX]; - int rc = fuse_materialize_fd(fd, materialized_path, - sizeof(materialized_path)); - if (rc < 0) { - if (replaced_remove_fd >= 0) - close(replaced_remove_fd); - return rc; - } - host_backing_fd = open(materialized_path, O_RDONLY | O_CLOEXEC); - int saved_errno = errno; - unlink(materialized_path); - if (host_backing_fd < 0) { - errno = saved_errno; - if (replaced_remove_fd >= 0) - close(replaced_remove_fd); - return linux_errno(); - } - close_host_backing_fd = true; + if (materialized_fd >= 0) { + host_backing_fd = materialized_fd; } else { int64_t ref_err = host_fd_ref_open(fd, &backing_ref); if (ref_err < 0) { @@ -1136,6 +2278,7 @@ static int64_t sys_mmap_high_va(guest_t *g, if (!host) goto fail; memset(host, 0, BLOCK_2MIB); + guest_dirty_clear_zeroed_range(g, gpa, gpa + BLOCK_2MIB); /* Detect freshness BEFORE guest_map_va_range so the decision is not * confused by a prior high-VA mmap into the same 2 MiB block. A fresh @@ -1214,28 +2357,11 @@ static int64_t sys_mmap_high_va(guest_t *g, } else if (prot != LINUX_PROT_NONE) { memset(map_host, 0, length); replaced_bytes_dirty = replacing_existing; - uint8_t *dst = map_host; - size_t remaining = length; - off_t file_off = (off_t) offset; - while (remaining > 0) { - ssize_t nr = pread(host_backing_fd, dst, remaining, file_off); - if (nr < 0) { - if (errno == EINTR) - continue; - - /* Real host I/O failure (not EINTR); previously the loop broke - * without setting ret and the syscall returned a "successful" - * partially-zero mapping. - */ - ret = linux_errno(); - goto fail; - } - if (nr == 0) - break; - dst += nr; - remaining -= (size_t) nr; - file_off += nr; - } + uint64_t gpa_for_addr = backing_gpa_start + (addr - va_base); + ret = read_file_range_to_guest(g, gpa_for_addr, host_backing_fd, offset, + length); + if (ret < 0) + goto fail; } /* Install L3 PTEs for the actual mapped range. Fresh blocks were fully @@ -1290,8 +2416,6 @@ static int64_t sys_mmap_high_va(guest_t *g, * steps must not goto fail or the region's backing fd would be * double-closed. */ - if (close_host_backing_fd && host_backing_fd >= 0) - close(host_backing_fd); host_fd_ref_close(&backing_ref); if (replaced_snaps) { close_region_snapshots(replaced_snaps, replaced_nsnaps); @@ -1390,8 +2514,6 @@ static int64_t sys_mmap_high_va(guest_t *g, } if (replaced_bytes_snap) free(replaced_bytes_snap); - if (close_host_backing_fd && host_backing_fd >= 0) - close(host_backing_fd); host_fd_ref_close(&backing_ref); /* Close the siblings_quiesced bracket as the very last step, so the byte @@ -1403,11 +2525,89 @@ static int64_t sys_mmap_high_va(guest_t *g, return ret; } +int mmap_prepare_file(guest_t *g, + uint64_t addr, + uint64_t length, + int flags, + int fd, + int64_t offset, + int *materialized_fd) +{ + assert(!mmap_lock_held_by_current_thread()); + *materialized_fd = -1; + if (flags & LINUX_MAP_ANONYMOUS) + return 0; + if (!length || (offset & (GUEST_PAGE_SIZE - 1))) + return -LINUX_EINVAL; + if (!fuse_fd_refuse_mmap(fd)) + return 0; + if (!g->is_rosetta || + !(flags & (LINUX_MAP_FIXED | LINUX_MAP_FIXED_NOREPLACE)) || + addr < g->guest_size || (flags & LINUX_MAP_SHARED)) + return -LINUX_ENODEV; + if (length > UINT64_MAX - (GUEST_PAGE_SIZE - 1)) + return -LINUX_ENOMEM; + length = PAGE_ALIGN_UP(length); + if (addr & (GUEST_PAGE_SIZE - 1)) + return -LINUX_EINVAL; + if (addr > 0x0000FFFFFFFFFFFFULL) + return -LINUX_ENOMEM; + + mmap_lock_acquire(g); + int rc = mmap_high_va_preflight_locked( + g, addr, length, true, (flags & LINUX_MAP_FIXED_NOREPLACE) != 0); + mmap_lock_release(); + if (rc < 0) + return rc; + + char path[PATH_MAX]; + rc = fuse_materialize_fd(fd, path, sizeof(path)); + if (rc < 0) + return rc; + int backing_fd = open(path, O_RDONLY | O_CLOEXEC); + int saved_errno = errno; + unlink(path); + if (backing_fd < 0) { + errno = saved_errno; + return linux_errno(); + } + *materialized_fd = backing_fd; + return 0; +} + +/* Page-table high-water slot (mmap_rx_end / mmap_end) for a TTBR0 mmap offset, + * or NULL if @off is not in either mmap arena. + */ +static uint64_t *mmap_pt_end_for_off(guest_t *g, uint64_t off) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) + return &g->mmap_rx_end; + if (off >= MMAP_BASE) + return &g->mmap_end; + return NULL; +} + +/* Advance the allocation high-water mark (mmap_rx_next / mmap_next) that fork + * IPC state transfer replays, for whichever arena @off belongs to. + */ +static void mmap_bump_next(guest_t *g, uint64_t off, uint64_t end) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) { + if (end > g->mmap_rx_next) + g->mmap_rx_next = end; + } else if (off >= MMAP_BASE) { + if (end > g->mmap_next) + g->mmap_next = end; + } +} + static int mremap_extend_range(guest_t *g, uint64_t off, uint64_t size, int prot) { + uint64_t *pt_end = mmap_pt_end_for_off(g, off); + if (prot == LINUX_PROT_NONE) { guest_invalidate_ptes(g, off, off + size); return 0; @@ -1418,10 +2618,67 @@ static int mremap_extend_range(guest_t *g, uint64_t ext_end = ALIGN_UP(off + size, BLOCK_2MIB); if (ext_end > g->guest_size) ext_end = g->guest_size; - if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) + size_t nblocks = pt_end ? (size_t) ((ext_end - ext_start) / BLOCK_2MIB) : 0; + bool *block_preexisting = NULL; + if (nblocks) { + block_preexisting = calloc(nblocks, sizeof(*block_preexisting)); + if (!block_preexisting) + return -1; + for (size_t i = 0; i < nblocks; i++) + block_preexisting[i] = + guest_va_block_mapped(g, ext_start + (uint64_t) i * BLOCK_2MIB); + } + if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) { + free(block_preexisting); return -1; - guest_update_perms(g, off, off + size, page_perms); + } + uint64_t saved_pt_end = pt_end ? *pt_end : 0; + if (pt_end && ext_end > *pt_end) + *pt_end = ext_end; + + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + uint64_t keep_start = off > b ? off : b; + uint64_t keep_end = off + size < bend ? off + size : bend; + if (keep_start <= b && keep_end >= bend) + continue; + if (guest_split_block(g, b) < 0) + goto fail; + if (b < keep_start && guest_invalidate_ptes(g, b, keep_start) < 0) + goto fail; + if (keep_end < bend && guest_invalidate_ptes(g, keep_end, bend) < 0) + goto fail; + } + + if (guest_update_perms(g, off, off + size, page_perms) < 0) + goto fail; + free(block_preexisting); return 0; + + /* Roll back: re-invalidate every fresh block whole. The forward pass + * already cleared the non-kept subranges. Also re-invalidate the kept [off, + * off+size) span, returning the range to its pre-extend state. + */ +fail: + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + (void) guest_invalidate_ptes(g, b, bend); + } + (void) guest_invalidate_ptes(g, off, off + size); + if (pt_end) + *pt_end = saved_pt_end; + free(block_preexisting); + return -1; } static int hvf_apply_file_overlay(guest_t *g, @@ -1452,6 +2709,12 @@ static int read_file_range_to_guest(guest_t *g, uint8_t *dst = host_ptr_for_gpa(g, gpa); if (!dst) return -LINUX_EFAULT; + + /* A short read, EOF, or later error may still leave nonzero bytes in the + * destination. Mark before the first pread so every exit is conservative. + */ + if (len <= UINT64_MAX - gpa) + guest_dirty_mark_range(g, gpa, gpa + len); size_t remaining = len; while (remaining > 0) { @@ -1631,6 +2894,25 @@ static void mark_mremap_source_overlay_metadata(guest_t *g, } } +/* Invalid pages in an accessible lazy source still contain retired slab bytes. + * Hold mmap_lock throughout and zero them before destination PTE creation can + * cover the source's edge block. Valid pages retain guest writes. + */ +static void zero_mremap_lazy_source(guest_t *g, uint64_t start, uint64_t end) +{ + while (start < end) { + if (guest_va_pte_valid(g, start)) { + start += GUEST_PAGE_SIZE; + continue; + } + uint64_t zero_start = start; + do { + start += GUEST_PAGE_SIZE; + } while (start < end && !guest_va_pte_valid(g, start)); + memset((uint8_t *) g->host_base + zero_start, 0, start - zero_start); + } +} + /* Copy each source segment according to its own backing state. Live-overlay * bytes must be refreshed from the file after the overlay is removed; private * fork-grown bytes remain in the slab and must be copied from their GPA. @@ -2018,12 +3300,16 @@ static int rollback_fresh_mmap_allocation(guest_t *g, { if (overlay_installed) hvf_remove_file_overlay(g, overlay_ipa, overlay_len); - if (guest_invalidate_ptes(g, start, start + length) < 0) + uint64_t end = start + length; + uint64_t cur_mmap_end = g->mmap_end; + uint64_t cur_mmap_rx_end = g->mmap_rx_end; + if (guest_invalidate_ptes(g, start, end) < 0) return -LINUX_ENOMEM; g->mmap_next = saved_mmap_next; - g->mmap_end = saved_mmap_end; + g->mmap_end = cur_mmap_end > saved_mmap_end ? cur_mmap_end : saved_mmap_end; g->mmap_rx_next = saved_mmap_rx_next; - g->mmap_rx_end = saved_mmap_rx_end; + g->mmap_rx_end = cur_mmap_rx_end > saved_mmap_rx_end ? cur_mmap_rx_end + : saved_mmap_rx_end; g->mmap_rw_gap_hint = saved_rw_gap_hint; g->mmap_rx_gap_hint = saved_rx_gap_hint; return 0; @@ -2372,6 +3658,8 @@ static int hvf_apply_file_overlay(guest_t *g, return -LINUX_EINTR; /* Being reaped; abandon the overlay */ int err = hvf_apply_file_overlay_quiesced(g, ipa, len, fd, file_off); thread_resume_siblings(); + if (err == 0 && len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); return err; } @@ -2394,6 +3682,13 @@ static int hvf_remove_file_overlay_quiesced(guest_t *g, return err; } + /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. The + * following munmap/MAP_FIXED path will clear the bit only after it has + * actually zeroed a complete 2 MiB block. + */ + if (len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); + for (int i = 0; i < nsegments; i++) { if (hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, segments[i].ipa, segments[i].len, @@ -2742,7 +4037,8 @@ int64_t sys_mmap(guest_t *g, int prot, int flags, int fd, - int64_t offset) + int64_t offset, + int materialized_fd) { bool is_anon = (flags & LINUX_MAP_ANONYMOUS) != 0; bool needs_exec = (prot & LINUX_PROT_EXEC) != 0; @@ -2752,6 +4048,20 @@ int64_t sys_mmap(guest_t *g, mmap_fresh_init(&fresh); int host_backing_fd = -1; + /* Anonymous mappings defer page-table creation and zeroing to first touch + * (guest fault or host-side access), like MAP_NORESERVE always has. This + * keeps mmap()/munmap() cost independent of length: a multi-GiB reservation + * costs neither an eager PTE walk nor a full-length memset, and + * never-touched blocks consume no page-table pool. PROT_NONE stays a pure + * reservation (faults deliver SIGSEGV, not materialization), and MAP_FIXED + * keeps the eager path because it must atomically replace live mappings. + * Shared anonymous memory stays eager unless the caller opted into + * MAP_NORESERVE (the historical lazy set), since deferred zeroing has never + * been exercised against the fork snapshot paths for it. + */ + bool is_lazy = is_anon && !is_prot_none && + ((flags & LINUX_MAP_SHARED) == 0 || is_noreserve); + /* Tracks whether hvf_apply_file_overlay has installed a host * MAP_FIXED|MAP_SHARED mapping that the failure paths must undo if later * steps (page tables, region tracking) fall over. Without this, a @@ -2776,13 +4086,25 @@ int64_t sys_mmap(guest_t *g, * free()'d (free(NULL) is a no-op) before return. */ bool replaced_regions_removed = false; + + /* Linux kernel rejects MAP_FIXED with non-page-aligned address (checked + * below); the flag itself is needed early because it gates the lazy path. + */ + bool is_fixed = + (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); + if (is_fixed) + is_lazy = false; int track_flags = ((flags & LINUX_MAP_SHARED) ? LINUX_MAP_SHARED : LINUX_MAP_PRIVATE); if (is_anon) track_flags |= LINUX_MAP_ANONYMOUS; - /* Preserve MAP_NORESERVE in region metadata before merge checks run. */ - if (is_noreserve) + /* Preserve MAP_NORESERVE in region metadata before merge checks run. The + * same bit doubles as the internal lazy marker: guest_region_add_ex derives + * the region's deferred-PTE flag from it, and it is not guest visible + * (/proc/self/maps prints only prot and shared/private). + */ + if (is_noreserve || is_lazy) track_flags |= LINUX_MAP_NORESERVE; /* The memory syscall layer handles all mmap variants. Aligned file-backed @@ -2800,7 +4122,9 @@ int64_t sys_mmap(guest_t *g, if (!is_anon && (offset & 4095)) return -LINUX_EINVAL; - if (!is_anon && fuse_fd_refuse_mmap(fd)) { + if (!is_anon && materialized_fd < 0 && fuse_fd_refuse_mmap(fd)) + return -LINUX_ENODEV; + if (materialized_fd >= 0) { bool allow_materialized_fuse_mmap = g->is_rosetta && ((flags & LINUX_MAP_FIXED) || @@ -2817,9 +4141,17 @@ int64_t sys_mmap(guest_t *g, if (length == 0) return -LINUX_ENOMEM; + /* A non-fixed nonzero address is a strong Linux hint. If it lands in the + * current (stopped) vCPU's invisible arena tail, release that tail before + * gap finding so implementation-only VA preparation does not perturb the + * address the application observes. Sibling arenas stay immutable without + * quiesce; their disjoint high-water placement makes self-overlap the + * normal and important case (allocator hinting near its previous result). + */ + if (!is_fixed && addr != 0) + mmap_fastpath_release_current_hint_locked(g, addr, length); + /* Linux kernel rejects MAP_FIXED with non-page-aligned address */ - bool is_fixed = - (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); if (is_fixed && (addr & 4095)) return -LINUX_EINVAL; @@ -2828,6 +4160,14 @@ int64_t sys_mmap(guest_t *g, */ bool is_noreplace = (flags & LINUX_MAP_FIXED_NOREPLACE) != 0; + /* A fixed mapping may replace an address previously handed out by an EL1 + * arena with a file/shared/stack-like mapping. Revoke all descriptors + * before making that semantic transition so a later fast munmap cannot + * classify it from the stale arena bounds. + */ + if (is_fixed) + mmap_fastpath_revoke_all_locked(g, false); + uint64_t result_off; /* Result as offset (0-based) */ if (is_fixed) { /* Addresses above TASK_SIZE (bit 63 set or beyond user VA range) are @@ -2838,7 +4178,7 @@ int64_t sys_mmap(guest_t *g, if (addr >= g->guest_size) return sys_mmap_high_va(g, addr, length, prot, flags, fd, offset, - true, is_noreplace); + true, is_noreplace, materialized_fd); /* High-VA MAP_FIXED (rosetta's JIT slabs at 240 TiB, code caches at 85 * TiB, etc.) is not safe to expose yet. The previous draft could @@ -2869,6 +4209,7 @@ int64_t sys_mmap(guest_t *g, uint64_t fix_end = off + length; if (guest_range_hits_infra(g, off, fix_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, off, fix_end); result_off = off; @@ -3110,7 +4451,7 @@ int64_t sys_mmap(guest_t *g, if (g->is_rosetta && addr >= g->guest_size && addr <= 0x0000FFFFFFFFFFFFULL) { int64_t high_hint = sys_mmap_high_va(g, addr, length, prot, flags, - fd, offset, false, false); + fd, offset, false, false, -1); if (high_hint >= 0) return high_hint; } @@ -3171,9 +4512,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rx_hwm = result_off + length; - if (rx_hwm > g->mmap_rx_next) - g->mmap_rx_next = rx_hwm; + mmap_bump_next(g, result_off, result_off + length); } else { /* RW (or PROT_NONE, or PROT_READ): allocate from main mmap region. * Honor the address hint if provided and within bounds. Some @@ -3232,9 +4571,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rw_hwm = result_off + length; - if (rw_hwm > g->mmap_next) - g->mmap_next = rw_hwm; + mmap_bump_next(g, result_off, result_off + length); } if (!region_has_capacity_after_removes(g, NULL, 0, 1)) { mmap_fresh_dispose(&fresh); @@ -3259,7 +4596,7 @@ int64_t sys_mmap(guest_t *g, guest_invalidate_ptes(g, result_off, result_off + length); } - if (!is_prot_none && !is_fixed && !is_noreserve) { + if (!is_prot_none && !is_fixed && !is_lazy) { /* Extend page tables for this specific allocation range only. * guest_extend_page_tables skips already-mapped blocks, so calling it * on pre-mapped regions is a no-op. This avoids creating entries for @@ -3317,16 +4654,24 @@ int64_t sys_mmap(guest_t *g, g->mmap_end = ext_end; } - /* Zero the mapped region */ + /* Zero the mapped region. RX mappings cannot be dirtied through their + * published PTEs, so a complete-block zero can make them clean again. + * Other mappings currently use writable stage-1 entries and must stay + * conservatively dirty even if their requested Linux prot is read-only. + */ memset((uint8_t *) g->host_base + result_off, 0, length); + if (needs_exec && !(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, result_off, result_off + length); } - /* MAP_NORESERVE: invalidate any stale PTEs (like PROT_NONE path) but track - * the region for lazy materialization on first fault. Page table entries - * will be created by guest_materialize_lazy() when the guest first touches - * a page in this range. + /* Lazy (private anonymous, incl. MAP_NORESERVE): invalidate any stale PTEs + * (like the PROT_NONE path) but track the region for lazy materialization + * on first fault. Page table entries will be created by + * guest_materialize_lazy() when the guest first touches a page in this + * range, or by the host-access fault-in path when a syscall targets the + * range before the guest ever touches it. */ - if (is_noreserve && !is_fixed) { + if (is_lazy) { guest_invalidate_ptes(g, result_off, result_off + length); } @@ -3392,6 +4737,7 @@ int64_t sys_mmap(guest_t *g, overlay_ipa = result_off; overlay_len = nf_overlay_len; } else { + guest_dirty_mark_range(g, result_off, result_off + length); uint8_t *dst = (uint8_t *) g->host_base + result_off; size_t remaining = length; off_t file_off = offset; @@ -3579,6 +4925,22 @@ int64_t sys_mremap(guest_t *g, if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; + /* Wait for both ranges before retaining source metadata: a condition wait + * releases mmap_lock and can invalidate region pointers and borrowed fds. + * Invalid fixed destinations keep their existing checks below. + */ + uint64_t wait_start = old_off, wait_end = old_off + old_size; + if ((flags & LINUX_MREMAP_FIXED) && new_addr >= g->ipa_base) { + uint64_t dest = new_addr - g->ipa_base; + if (dest <= g->guest_size && new_size <= g->guest_size - dest) { + if (dest < wait_start) + wait_start = dest; + if (dest + new_size > wait_end) + wait_end = dest + new_size; + } + } + guest_materialize_wait_range_locked(g, wait_start, wait_end); + /* Verify the whole source range is covered by one logical VMA. A fork-aware * growth can split that VMA at the inherited/private boundary, but no * unrelated adjacent mapping may be included. @@ -3628,10 +4990,13 @@ int64_t sys_mremap(guest_t *g, /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ - memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, - tail_end - tail_off); + uint64_t tail_gpa = src_gpa_base + (tail_off - src_start); + if (guest_invalidate_ptes(g, tail_off, tail_end) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); + memset(host_ptr_for_gpa(g, tail_gpa), 0, tail_end - tail_off); + guest_dirty_clear_zeroed_range(g, tail_gpa, + tail_gpa + (tail_end - tail_off)); guest_region_remove_reserved(g, tail_off, tail_end, tail_remove_fd); - guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) @@ -3639,6 +5004,12 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, (int64_t) old_addr); } + bool source_lazy = + src_reg->noreserve && (src_reg->flags & LINUX_MAP_ANONYMOUS) && + src_reg->prot != LINUX_PROT_NONE && src_gpa_base == src_start && + old_off < g->guest_size && old_size <= g->guest_size - old_off && + !mremap_source_has_overlay(&source); + /* MREMAP_FIXED: move to a specific new address */ if (flags & LINUX_MREMAP_FIXED) { if (new_addr & 4095) @@ -3658,7 +5029,6 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) return finish_mremap(&source, -LINUX_EINVAL); - /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; if (old_off < new_end && new_off < old_end) @@ -3790,6 +5160,9 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, cleanup_err); } + uint64_t copy_len = old_size < new_size ? old_size : new_size; + if (source_lazy) + zero_mremap_lazy_source(g, old_off, old_off + copy_len); if (mremap_extend_range(g, new_off, new_size, move.track.prot) < 0) { int restore_err = restore_snapshot_overlays_in_place( g, move.source_snaps, move.source_nsnaps); @@ -3799,7 +5172,14 @@ int64_t sys_mremap(guest_t *g, } (void) restore_snapshot_overlays_in_place(g, move.dest_snaps, move.dest_nsnaps); + int pt_err = + restore_snapshot_page_tables(g, new_off, new_off + new_size, + move.dest_snaps, move.dest_nsnaps); + if (pt_err < 0) + restore_err = pt_err; mremap_move_dispose(&move); + if (restore_err < 0) + return finish_mremap(&source, restore_err); return finish_mremap(&source, -LINUX_ENOMEM); } @@ -3815,7 +5195,6 @@ int64_t sys_mremap(guest_t *g, * overlay reapplied), and msync's emulated pwrite-the-diff path keeps * subsequent writes consistent. */ - uint64_t copy_len = old_size < new_size ? old_size : new_size; if (move.track.prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); } else { @@ -3856,14 +5235,22 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (move.track.prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping */ if (old_size > 0) { - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove_reserved(g, old_off, old_off + old_size, move.source_remove_fd); move.source_remove_fd = -1; - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3933,6 +5320,12 @@ int64_t sys_mremap(guest_t *g, mremap_track_dispose(&track); return finish_mremap(&source, -LINUX_ENOMEM); } + if (source_lazy) { + uint64_t tail = ALIGN_DOWN(grow_off, BLOCK_2MIB); + if (tail < old_off) + tail = old_off; + zero_mremap_lazy_source(g, tail, grow_off); + } if (mremap_extend_range(g, grow_off, grow_len, track.prot) < 0) { mremap_track_dispose(&track); @@ -3942,6 +5335,9 @@ int64_t sys_mremap(guest_t *g, } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); + if (!(track.prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, grow_off, + grow_off + grow_len); /* Update region tracking: remove old, add extended */ guest_region_remove_reserved(g, old_off, old_off + old_size, @@ -3954,14 +5350,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, old_off, old_off + new_size); /* Update high-water marks */ - uint64_t hwm = old_off + new_size; - if (old_off >= MMAP_RX_BASE && old_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (old_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, old_off, old_off + new_size); return finish_mremap(&source, (int64_t) old_addr); } @@ -4034,6 +5423,8 @@ int64_t sys_mremap(guest_t *g, } } + if (source_lazy) + zero_mremap_lazy_source(g, old_off, old_off + old_size); if (mremap_extend_range(g, new_off, new_size, track.prot) < 0) { if (source_overlay) { int restore_err = @@ -4080,14 +5471,22 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (track.prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping. Any live source overlay was already torn down * before the destination range was touched. */ - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove_reserved(g, old_off, old_off + old_size, source_remove_fd); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -4100,14 +5499,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, new_off, new_off + new_size); /* Update high-water marks */ - uint64_t hwm = new_off + new_size; - if (new_off >= MMAP_RX_BASE && new_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (new_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, new_off, new_off + new_size); return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } @@ -4197,6 +5589,8 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) */ if (!madvise_range_mapped(g, off, length)) return -LINUX_ENOMEM; + if (in_primary) + guest_materialize_wait_range_locked(g, off, off + length); uint64_t end = off + length; for (int i = 0; i < g->nregions; i++) { @@ -4297,7 +5691,7 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) int64_t sys_mmap_anon(guest_t *g, uint64_t addr, uint64_t length, int prot) { return sys_mmap(g, addr, length, prot, - LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, -1, 0); + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, -1, 0, -1); } static int compare_range_pair(const void *a, const void *b) @@ -4312,11 +5706,38 @@ static int compare_range_pair(const void *a, const void *b) return 0; } +/* Coalesced sub-ranges of a munmap that must be zeroed. Sized so that even a + * pathologically fragmented lazy mapping (alternating materialized and + * untouched blocks) rarely overflows; on overflow the remainder of the region + * overlap is zeroed wholesale, which is always correct (zeroing already-zero + * slab bytes), just slower. + */ +#define MUNMAP_ZERO_RANGES_MAX 128 + +typedef struct { + uint64_t lo, hi; +} zero_range_t; + +static void zero_range_push(zero_range_t *ranges, + int *n, + uint64_t lo, + uint64_t hi) +{ + if (lo >= hi) + return; + if (*n > 0 && ranges[*n - 1].hi == lo) { + ranges[*n - 1].hi = hi; + return; + } + ranges[(*n)++] = (zero_range_t) {lo, hi}; +} + static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) { /* Reject munmap targeting VM infrastructure regions. */ if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, unmap_off, end); /* An interior removal from a file-backed region needs a second owned fd for * the surviving right half. Reserve it before changing overlays, page @@ -4337,26 +5758,91 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) return cleanup_err; } - /* Invalidate PTEs first. This may need to split a 2MiB block which can fail - * if the page table pool is exhausted. Failing before region removal keeps - * metadata consistent. + /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below + * destroys the evidence. A pure anonymous, private, non-overlaid region + * needs no eager zeroing at all: its dirty bitmap already reflects every + * write that ever touched it. The same guarantee + * munmap_retire_commit_locked() relies on to skip zeroing entirely on the + * EL1 fast munmap path, gated by the identical + * anonymous/private/no-backing-fd/no-overlay check below. Also, + * hvf_remove_file_overlay_quiesced() explicitly marks a restored overlay's + * backing dirty via guest_dirty_mark_range() before this function ever sees + * it. guest_materialize_lazy_one() zeros dirty backing before publishing + * any new descriptor, so deferring here costs nothing at reuse time either; + * it only stops paying up front to zero bytes a future mapping may never + * touch. + * + * An actual file-backed region, or one whose overlay cleanup above could + * not tear down, keeps the eager policy below: eager regions are zeroed + * across the whole overlap, and lazy (deferred-PTE, MAP_NORESERVE) regions + * only need their materialized 2MiB blocks zeroed. This keeps that + * fallback's cost proportional to memory actually touched instead of to the + * mapping length. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) { - if (remove_fd >= 0) - close(remove_fd); - return -LINUX_ENOMEM; - } + zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; + int nzr = 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (r->start >= end) break; if (r->end <= unmap_off) continue; if (r->prot == LINUX_PROT_NONE) continue; + if ((r->flags & LINUX_MAP_ANONYMOUS) && + !(r->flags & LINUX_MAP_SHARED) && r->backing_fd < 0 && + !r->overlay_active) + continue; uint64_t zstart = (r->start > unmap_off) ? r->start : unmap_off; uint64_t zend = (r->end < end) ? r->end : end; - memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); + if (!r->noreserve) { + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: widen the last range instead of dropping any + * span that must be zeroed. Everything between ranges lies + * inside [unmap_off, end) and is being unmapped, so zeroing the + * gap as well is harmless. + */ + zr[nzr - 1].hi = zend; + continue; + } + zero_range_push(zr, &nzr, zstart, zend); + continue; + } + for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { + if (!guest_va_block_mapped(g, b)) { + /* Jump through the PTE occupancy index to the next materialized + * block. A huge untouched reservation therefore does no work + * proportional to its virtual length. + */ + b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); + continue; + } + uint64_t lo = (b > zstart) ? b : zstart; + uint64_t hi = (b + BLOCK_2MIB < zend) ? b + BLOCK_2MIB : zend; + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: fold the remainder of this overlap into the + * last range and stop scanning blocks. + */ + zr[nzr - 1].hi = zend; + break; + } + zero_range_push(zr, &nzr, lo, hi); + b += BLOCK_2MIB; + } + } + + /* Invalidate PTEs first. This may need to split a 2MiB block which can fail + * if the page table pool is exhausted. Failing before region removal keeps + * metadata consistent. + */ + if (guest_invalidate_ptes(g, unmap_off, end) < 0) { + if (remove_fd >= 0) + close(remove_fd); + return -LINUX_ENOMEM; + } + for (int i = 0; i < nzr; i++) { + memset((uint8_t *) g->host_base + zr[i].lo, 0, zr[i].hi - zr[i].lo); + guest_dirty_clear_zeroed_range(g, zr[i].lo, zr[i].hi); } guest_region_remove_reserved(g, unmap_off, end, remove_fd); if (unmap_off < g->mmap_rw_gap_hint) @@ -4381,7 +5867,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) if (nranges <= 0) return; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < nranges; i++) { int rc = munmap_guest_range(g, starts[i], ends[i]); if (rc < 0) { @@ -4394,7 +5880,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) } thread_drop_deferred_stack_unmap(t, starts[i], ends[i]); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* sys_munmap. */ @@ -4469,6 +5955,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) thread_finish_deferred_stack_ranges(txns, nranges); } } + mmap_fastpath_rewind_current_if_clean_locked(g); return 0; } @@ -4496,6 +5983,12 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (addr > UINT64_MAX - length) return -LINUX_EINVAL; + /* Permission and VMA-shape changes are slow-path boundaries. Retire any + * already-published unmaps, then invalidate arena generations before the + * metadata/PTE edit so EL1 cannot act on the old anonymous classification. + */ + mmap_fastpath_revoke_all_locked(g, false); + if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { uint64_t mprot_end = addr + length; @@ -4551,6 +6044,7 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) */ if (guest_range_hits_infra(g, mprot_off, mprot_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, mprot_off, mprot_end); /* Same max_prot check as the high-VA branch above. */ if ((prot & LINUX_PROT_WRITE) && @@ -4568,6 +6062,19 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (prot != LINUX_PROT_NONE) { int page_perms = prot_to_perms(prot); + + /* Materialize lazy blocks in the range at their region's + * current prot before the block-granular extend below. The + * extend stamps whole 2MiB blocks with page_perms; on an + * unmaterialized lazy region that would hand every neighbor + * page OUTSIDE [mprot_off, mprot_end) the sub-range's + * permissions (region says RW, PTE says R-only, host-side + * writes EFAULT). guest_materialize_lazy covers block-within- + * region at region prot, so after this the extend is a no-op + * for lazy regions and update_perms below adjusts only the + * requested range. + */ + guest_lazy_faultin_locked(g, mprot_off, mprot_end - mprot_off); if (guest_extend_page_tables(g, mprot_off, mprot_end, page_perms) < 0) return -LINUX_ENOMEM; @@ -4804,12 +6311,12 @@ int64_t sys_msync(guest_t *g, uint64_t addr, uint64_t length, int flags) int *fsync_fds = NULL; int fsync_count = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire_raw(); uint64_t cursor = off; while (cursor < end) { const guest_region_t *r = guest_region_find(g, cursor); if (!r || r->start > cursor) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release_raw(); return -LINUX_ENOMEM; } cursor = r->end < end ? r->end : end; @@ -4818,7 +6325,7 @@ int64_t sys_msync(guest_t *g, uint64_t addr, uint64_t length, int flags) if (flags & LINUX_MS_SYNC) { fsync_fds = calloc((size_t) g->nregions, sizeof(*fsync_fds)); if (!fsync_fds) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release_raw(); return -LINUX_ENOMEM; } } @@ -4915,7 +6422,7 @@ int64_t sys_msync(guest_t *g, uint64_t addr, uint64_t length, int flags) if (ret < 0) break; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release_raw(); /* Every queued fd is fsynced unconditionally, even if a later region's * diff/refresh failed and broke the locked loop early: the original code @@ -4967,7 +6474,14 @@ int mmap_fork_prepare_anon_shared(guest_t *g, if (!txn) return -LINUX_ENOMEM; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire_for_fork(g); + + /* fork callers have quiesced siblings. Drain their last publications, + * revoke every descriptor, and trim never-consumed arena tails before the + * legacy [MMAP_BASE,mmap_next) snapshot range is computed. + */ + mmap_fastpath_revoke_all_locked(g, true); + guest_materialize_wait_all_locked(g); size_t hps = host_page_size_cached(); @@ -5112,7 +6626,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return -LINUX_ENOMEM; } @@ -5125,7 +6639,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return nsnaps; } @@ -5165,7 +6679,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, close(fd); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return 0; } @@ -5184,7 +6698,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, mmap_fork_anon_shared_txn_t *txn = *txn_ptr; int rc = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = txn->noverlays - 1; i >= 0; i--) { const fork_overlay_snapshot_t *ovl = &txn->overlays[i]; @@ -5254,7 +6768,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); mmap_fork_dispose_anon_shared_txn(txn_ptr); return rc; } @@ -5268,7 +6782,7 @@ int mmap_fork_restore_overlays(guest_t *g, const uint64_t *parent_ovl_start, const uint64_t *parent_ovl_end) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int rc = 0; for (int i = 0; i < g->nregions; i++) { @@ -5357,6 +6871,6 @@ int mmap_fork_restore_overlays(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return rc; } diff --git a/src/syscall/mem.h b/src/syscall/mem.h index e90312cb..72c800a9 100644 --- a/src/syscall/mem.h +++ b/src/syscall/mem.h @@ -21,14 +21,26 @@ typedef struct mmap_fork_anon_shared_txn mmap_fork_anon_shared_txn_t; /* brk: set/query program break */ int64_t sys_brk(guest_t *g, uint64_t addr); -/* mmap: map pages into guest address space */ +/* Prepare FUSE backing without mmap_lock held. On success the caller owns + * materialized_fd, or receives -1 when the ordinary fd path applies. + */ +int mmap_prepare_file(guest_t *g, + uint64_t addr, + uint64_t length, + int flags, + int fd, + int64_t offset, + int *materialized_fd); + +/* Borrows materialized_fd (-1 for ordinary mappings) with mmap_lock held. */ int64_t sys_mmap(guest_t *g, uint64_t addr, uint64_t length, int prot, int flags, int fd, - int64_t offset); + int64_t offset, + int materialized_fd); /* munmap: unmap pages from guest address space */ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length); diff --git a/src/syscall/netlink.c b/src/syscall/netlink.c index 714dd03f..d103b725 100644 --- a/src/syscall/netlink.c +++ b/src/syscall/netlink.c @@ -814,7 +814,7 @@ static int64_t nl_timeout_store(guest_t *g, return -LINUX_EINVAL; nl_timeval_t tv; - if (guest_read_small(g, optval_gva, &tv, sizeof(tv)) < 0) + if (guest_read_nofault(g, optval_gva, &tv, sizeof(tv)) < 0) return -LINUX_EFAULT; if (tv.tv_usec < 0 || tv.tv_usec >= 1000000) @@ -1048,6 +1048,9 @@ int64_t netlink_setsockopt(guest_t *g, if (level != LINUX_SOL_SOCKET) return -LINUX_ENOPROTOOPT; + uint32_t need = nl_optlen_min(optname); + if (optlen >= sizeof(int32_t)) + (void) guest_lazy_faultin(g, optval_gva, optlen < need ? optlen : need); pthread_mutex_lock(&nl_lock); netlink_state_t *ns = nl_find(guest_fd); if (!ns) { @@ -1066,7 +1069,7 @@ int64_t netlink_setsockopt(guest_t *g, return -LINUX_EINVAL; } int32_t value = 0; - if (guest_read_small(g, optval_gva, &value, sizeof(value)) < 0) { + if (guest_read_nofault(g, optval_gva, &value, sizeof(value)) < 0) { pthread_mutex_unlock(&nl_lock); return -LINUX_EFAULT; } @@ -1111,7 +1114,7 @@ int64_t netlink_setsockopt(guest_t *g, break; case LINUX_SO_LINGER: { int32_t ling[2] = {0, 0}; - if (guest_read_small(g, optval_gva, ling, sizeof(ling)) < 0) { + if (guest_read_nofault(g, optval_gva, ling, sizeof(ling)) < 0) { ret = -LINUX_EFAULT; break; } @@ -1480,6 +1483,19 @@ static int64_t nl_msg_iovcnt(const linux_msghdr_t *mhdr, int *iovcnt) return 0; } +static void nl_prefault_iov(guest_t *g, + const linux_iovec_t *iov, + int iovcnt, + uint64_t limit) +{ + for (int i = 0; i < iovcnt && limit; i++) { + uint64_t len = iov[i].iov_len < limit ? iov[i].iov_len : limit; + if (len) + (void) guest_lazy_faultin(g, iov[i].iov_base, len); + limit -= len; + } +} + /* The send half of sendmsg(2), sendto(2), write(2) and writev(2) on a netlink * socket. * @@ -1494,6 +1510,7 @@ static int64_t netlink_send_iov(int guest_fd, const linux_iovec_t *iov, int iovcnt) { + nl_prefault_iov(g, iov, iovcnt, NETLINK_REQ_MAX); pthread_mutex_lock(&nl_lock); netlink_state_t *ns = nl_find(guest_fd); if (!ns) { @@ -1557,8 +1574,8 @@ static int64_t netlink_send_iov(int guest_fd, uint8_t req[NETLINK_REQ_MAX] = {0}; size_t rlen = 0; for (int i = 0; i < iovcnt; i++) { - if (guest_read(g, iov[i].iov_base, req + rlen, - (size_t) iov[i].iov_len) < 0) { + if (guest_read_nofault(g, iov[i].iov_base, req + rlen, + (size_t) iov[i].iov_len) < 0) { result = -LINUX_EFAULT; goto out; } @@ -1865,6 +1882,7 @@ static int64_t netlink_recv_iov(int guest_fd, int flags) { bool nonblock = (flags & LINUX_MSG_DONTWAIT) || fd_guest_nonblock(guest_fd); + nl_prefault_iov(g, iov, iovcnt, NETLINK_BUF_SIZE); pthread_mutex_lock(&nl_lock); netlink_state_t *ns = nl_find(guest_fd); if (!ns) { @@ -1906,8 +1924,8 @@ static int64_t netlink_recv_iov(int guest_fd, * chunk by chunk and a chunk that faults still places the bytes ahead * of it, which is what copy_to_iter() counts. */ - size_t moved = guest_write_partial(g, iov[i].iov_base, - ns->buf + ns->buf_pos, chunk); + size_t moved = guest_write_partial_nofault( + g, iov[i].iov_base, ns->buf + ns->buf_pos, chunk); ns->buf_pos += moved; done += moved; if (moved < chunk) { diff --git a/src/syscall/proc.c b/src/syscall/proc.c index fe8d1946..cea69f92 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -37,6 +37,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/futex.h" @@ -80,6 +81,23 @@ static _Atomic bool rosetta_enabled = true; */ static _Atomic bool rosetta_active = false; +static bool vcpu_exit_is_fork_family_syscall(hv_vcpu_t vcpu, + const hv_vcpu_exit_t *vexit) +{ + if (vexit->reason != HV_EXIT_REASON_EXCEPTION) + return false; + + uint64_t syndrome = vexit->exception.syndrome; + uint32_t ec = (uint32_t) ((syndrome >> 26) & 0x3f); + uint16_t imm = (uint16_t) (syndrome & 0xffff); + if (ec != 0x16 || imm != 5) + return false; + + uint64_t nr = 0; + hv_vcpu_get_reg(vcpu, HV_REG_X8, &nr); + return nr == SYS_clone || nr == SYS_clone3; +} + /* Process table for tracking direct and adopted fork children. Start small so * lifecycle tests exercise growth deterministically; expand under pid_lock as * the fork family grows. No pointer into this array survives unlocking. @@ -2765,6 +2783,9 @@ int64_t sys_waitid(guest_t *g, return result; } + if (infop_gva) + (void) guest_lazy_faultin(g, infop_gva, SIGINFO_SIZE); + /* Search process table for matching entry. P_ALL must scan all children * (not block on the first non-exited one), so the wait loop always use * WNOHANG in the inner loop and retry with timedwait if the caller @@ -2880,7 +2901,7 @@ int64_t sys_waitid(guest_t *g, memcpy(si + SIGINFO_OFF_UID, &uid, 4); memcpy(si + SIGINFO_OFF_STATUS, &si_status, 4); - if (guest_write_small(g, infop_gva, si, SIGINFO_SIZE) < 0) { + if (guest_write_nofault(g, infop_gva, si, SIGINFO_SIZE) < 0) { pthread_mutex_unlock(&pid_lock); return -LINUX_EFAULT; } @@ -3574,7 +3595,7 @@ static bool vcpu_handle_wx_toggle(guest_t *g, /* Hold mmap_lock for page table modifications AND region lookups to prevent * races with concurrent mmap/mprotect/munmap from other vCPU threads. */ - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire_raw(); /* Check if this is a genuine permission violation (not a W^X toggle). If * the guest region lacks the required permission, deliver SIGSEGV instead @@ -3586,7 +3607,7 @@ static bool vcpu_handle_wx_toggle(guest_t *g, const guest_region_t *reg = guest_region_find(g, off); int required = (type == 1) ? LINUX_PROT_WRITE : LINUX_PROT_EXEC; if (reg && !(reg->prot & required)) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release_raw(); uint64_t esr; hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_ESR_EL1, &esr); signal_set_fault_info(LINUX_SEGV_ACCERR, far, esr); @@ -3612,7 +3633,7 @@ static bool vcpu_handle_wx_toggle(guest_t *g, uint64_t block_start = far & ~(BLOCK_2MIB - 1); int sr = guest_split_block(g, block_start); int ur = guest_update_perms(g, page_start, page_end, new_perms); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release_raw(); if (verbose && (sr < 0 || ur < 0)) log_warn( "%s: W^X toggle FAILED " @@ -3841,9 +3862,9 @@ static bool vcpu_handle_el0_fault(guest_t *g, uint32_t fsc_type = (fsc >> 2) & 0xF; if (fsc_type == 0x01) { uint64_t fault_off = far_addr - g->ipa_base; - pthread_mutex_lock(&mmap_lock); - int mat = guest_materialize_lazy(g, fault_off); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + int mat = guest_materialize_lazy_fault(g, fault_off); + mmap_lock_release(); if (mat == 0) { /* Page materialized; the helpers inside guest_materialize_lazy * populated the per-vCPU TLBI accumulator with the range just @@ -3854,6 +3875,21 @@ static bool vcpu_handle_el0_fault(guest_t *g, * (negative) entries would re-fault on the retry, looping until the * entry self-evicts. */ + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_MATERIALIZE); + switch ((tlbi_kind_t) cpu_tlbi_req.kind) { + case TLBI_RANGE: + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_TLBI_VAE); + break; + case TLBI_RANGE_LARGE: + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_TLBI_RVAE); + break; + case TLBI_BROADCAST: + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_TLBI_BCAST); + break; + case TLBI_NONE: + default: + break; + } tlbi_request_emit_to_vcpu(vcpu); return true; } @@ -3900,9 +3936,9 @@ static bool vcpu_handle_el0_fault(guest_t *g, uint64_t live_avail = 0; void *live_pt = NULL; if (stale_plausible) { - pthread_mutex_lock(&mmap_lock); - live_pt = guest_ptr_avail(g, far_addr, &live_avail, want_perm); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + live_pt = guest_ptr_avail_nofault(g, far_addr, &live_avail, want_perm); + mmap_lock_release(); } if (live_pt) { /* Bound per vCPU and per (page, faulting PC). A genuinely stuck entry @@ -4710,12 +4746,39 @@ int vcpu_run_loop_with_hooks(hv_vcpu_t vcpu, HV_CHECK_CTX(hv_vcpu_run(vcpu), vcpu, g); + drain_external_guest_signal(); + + /* An HVF return can land inside the EL1 fast path's producer window, + * with retire.producer_active published for this vCPU's own slot. Only + * the guest clears that word, and every host path that takes mmap_lock + * waits for it in mmap_fastpath_host_gate_close(). Waiting there would + * block this thread on a vCPU that cannot run until this very thread + * re-enters it. Resume before dispatching either a cancellation or an + * exception. The producer window is bounded and non-blocking, so it + * retires at once; pending host attention survives to the next exit. + */ + while ((vexit->reason == HV_EXIT_REASON_CANCELED || + vexit->reason == HV_EXIT_REASON_EXCEPTION) && + mmap_fastpath_current_producer_active(g)) { + HV_CHECK_CTX(hv_vcpu_run(vcpu), vcpu, g); + drain_external_guest_signal(); + } + + /* Every return from HVF is a natural retirement point. Drain before + * dispatching syscalls, page faults, MAP_FIXED, fork/exec, signals, or + * exit so no host path can consult pre-munmap region metadata and + * rematerialize an EL1-invalidated page. The helper also drains mmap + * publications before the acquire-snapshotted retire entries. + */ + if (!mmap_fastpath_current_producer_active(g)) + mmap_fastpath_drain_vmexit( + g, vcpu_exit_is_fork_family_syscall(vcpu, vexit)); + + /* Main: disarm timeout */ if (is_main) atomic_store_explicit(&g_vcpu_progress, iter * 2 + 2, memory_order_relaxed); - drain_external_guest_signal(); - /* Re-check exit_group after waking from hv_vcpu_run */ if (proc_exit_group_requested()) { exit_code = proc_exit_group_code(); diff --git a/src/syscall/signal.c b/src/syscall/signal.c index cd5f2df9..b091c7e9 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -60,12 +60,14 @@ #include "hvutil.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/thread.h" #include "syscall/linux-wire.h" -#include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/internal.h" #include "syscall/proc.h" /* proc_get_pid, proc_get_uid, SYSCALL_EXEC_HAPPENED */ #include "proved/sigframe.h" #include "syscall/signal.h" @@ -1542,12 +1544,16 @@ int64_t signal_rt_sigaction(guest_t *g, int idx = signum - 1; bool reap_exited_sigchld = false; + if (act_gva) + (void) guest_lazy_faultin(g, act_gva, sizeof(linux_sigaction_t)); + if (oldact_gva) + (void) guest_lazy_faultin(g, oldact_gva, sizeof(linux_sigaction_t)); pthread_mutex_lock(&sig_lock); /* Return old action if requested */ if (oldact_gva) { - if (guest_write_small(g, oldact_gva, &sig_state.actions[idx], - sizeof(linux_sigaction_t)) < 0) { + if (guest_write_nofault(g, oldact_gva, &sig_state.actions[idx], + sizeof(linux_sigaction_t)) < 0) { pthread_mutex_unlock(&sig_lock); return -LINUX_EFAULT; } @@ -1556,7 +1562,7 @@ int64_t signal_rt_sigaction(guest_t *g, /* Install new action if provided */ if (act_gva) { linux_sigaction_t act; - if (guest_read_small(g, act_gva, &act, sizeof(act)) < 0) { + if (guest_read_nofault(g, act_gva, &act, sizeof(act)) < 0) { pthread_mutex_unlock(&sig_lock); return -LINUX_EFAULT; } @@ -1602,6 +1608,10 @@ int64_t signal_rt_sigprocmask(guest_t *g, if (sigsetsize != 8) return -LINUX_EINVAL; + if (set_gva) + (void) guest_lazy_faultin(g, set_gva, sizeof(uint64_t)); + if (oldset_gva) + (void) guest_lazy_faultin(g, oldset_gva, sizeof(uint64_t)); pthread_mutex_lock(&sig_lock); _Atomic uint64_t *blocked = thread_blocked_ptr(); @@ -1611,7 +1621,8 @@ int64_t signal_rt_sigprocmask(guest_t *g, */ if (oldset_gva) { uint64_t old_mask = atomic_load_explicit(blocked, memory_order_relaxed); - if (guest_write_small(g, oldset_gva, &old_mask, sizeof(old_mask)) < 0) { + if (guest_write_nofault(g, oldset_gva, &old_mask, sizeof(old_mask)) < + 0) { pthread_mutex_unlock(&sig_lock); return -LINUX_EFAULT; } @@ -1620,7 +1631,7 @@ int64_t signal_rt_sigprocmask(guest_t *g, /* Apply new mask if provided */ if (set_gva) { uint64_t set; - if (guest_read_small(g, set_gva, &set, sizeof(set)) < 0) { + if (guest_read_nofault(g, set_gva, &set, sizeof(set)) < 0) { pthread_mutex_unlock(&sig_lock); return -LINUX_EFAULT; } @@ -2095,6 +2106,14 @@ int64_t signal_sigaltstack(guest_t *g, uint64_t ss_gva, uint64_t old_ss_gva) */ if (ss.ss_sp > UINT64_MAX - ss.ss_size) return -LINUX_EINVAL; + + /* Alternate stacks have the same lifetime sensitivity as clone + * stacks: once registered, their unmap must take the host path + * instead of being classified only as an anonymous arena range. + */ + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); t->altstack_sp = ss.ss_sp; t->altstack_flags = 0; t->altstack_size = ss.ss_size; @@ -2231,7 +2250,8 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, pthread_mutex_unlock(&sig_lock); return 0; } - linux_sigaction_t *act = &sig_state.actions[idx]; + linux_sigaction_t action = sig_state.actions[idx]; + const linux_sigaction_t *act = &action; /* Check handler type */ if (act->sa_handler == LINUX_SIG_IGN) { @@ -2261,6 +2281,15 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, } } + /* Snapshot and reset the action before dropping sig_lock. Guest-memory + * access below may fault in lazy pages and acquire mmap_lock. + */ + if (act->sa_flags & LINUX_SA_RESETHAND) { + sig_state.actions[idx].sa_handler = LINUX_SIG_DFL; + sig_state.actions[idx].sa_flags &= ~LINUX_SA_SIGINFO; + } + pthread_mutex_unlock(&sig_lock); + /* Deliver to user handler: build rt_sigframe on guest stack */ /* Past every disposition that discards the signal, so this is the point a @@ -2314,7 +2343,6 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_ELR_EL1, saved_pc); if (rseq_rc == -1) { *exit_code = 128 + 11; /* SIGSEGV */ - pthread_mutex_unlock(&sig_lock); return -1; } } @@ -2428,7 +2456,6 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, (unsigned long long) (use_altstack && thr ? thr->altstack_sp : 0), signum); *exit_code = 128 + signum; - pthread_mutex_unlock(&sig_lock); return -1; } @@ -2454,7 +2481,6 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, if (pushed_cookie) sigreturn_cookie_depth--; *exit_code = 128 + signum; - pthread_mutex_unlock(&sig_lock); return -1; } @@ -2513,12 +2539,6 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, if (use_altstack && thr) thr->on_altstack = true; - /* 7. Reset to SIG_DFL if SA_RESETHAND is set */ - if (act->sa_flags & LINUX_SA_RESETHAND) { - act->sa_handler = LINUX_SIG_DFL; - act->sa_flags &= ~LINUX_SA_SIGINFO; - } - /* If delivery happens while returning from the syscall HVC path, the shim * still has the interrupted syscall frame on its EL1 stack. Tell it to drop * that frame so the handler PC/SP/LR/args installed above are not @@ -2531,7 +2551,6 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, if (!el0_preempt) hv_vcpu_set_reg(vcpu, HV_REG_X8, 2); - pthread_mutex_unlock(&sig_lock); return 1; } @@ -2549,6 +2568,7 @@ int signal_take_termination_wait_status(void) return status; } + /* signal_deliver_one() consumed a signal the guest never observes, so the * caller should look at the next one. Distinct from the documented 0/1/-1 * contract of deliver_signal_locked() and never escapes signal_deliver(). diff --git a/src/syscall/sys.c b/src/syscall/sys.c index 1b5755bc..679fbdd1 100644 --- a/src/syscall/sys.c +++ b/src/syscall/sys.c @@ -267,6 +267,9 @@ int64_t sys_getrandom(guest_t *g, int64_t sys_getcwd(guest_t *g, uint64_t buf_gva, uint64_t size) { + if (size) + (void) guest_lazy_faultin( + g, buf_gva, size < LINUX_PATH_MAX ? size : LINUX_PATH_MAX); proc_cwd_view_t view; if (proc_acquire_cwd_view(&view) < 0) return linux_errno(); @@ -277,7 +280,7 @@ int64_t sys_getcwd(guest_t *g, uint64_t buf_gva, uint64_t size) return -LINUX_ERANGE; } - int rc = guest_write_small(g, buf_gva, view.path, write_len); + int rc = guest_write_nofault(g, buf_gva, view.path, write_len); proc_release_cwd_view(&view); if (rc < 0) return -LINUX_EFAULT; diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index 12da9191..33a81674 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -73,6 +73,7 @@ #include "syscall/time.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "debug/syscall-hist.h" @@ -183,9 +184,9 @@ typedef int64_t (*syscall_handler_t)(guest_t *g, { \ (void) g; (void) x0; (void) x1; (void) x2; \ (void) x3; (void) x4; (void) x5; (void) verbose; \ - pthread_mutex_lock(&mmap_lock); \ + mmap_lock_acquire(g); \ int64_t r = (body); \ - pthread_mutex_unlock(&mmap_lock); \ + mmap_lock_release(); \ return r; \ } @@ -502,16 +503,16 @@ static void sc_sync_regions_inline(guest_t *g) * position) cannot make us skip an entry permanently. */ for (int i = 0;; i++) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); if (i >= g->nregions) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); break; } const guest_region_t *r = &g->regions[i]; int duped = -1; if (r->shared && r->backing_fd >= 0) duped = dup(r->backing_fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (duped < 0) continue; (void) fsync(duped); @@ -542,7 +543,7 @@ static int64_t sc_sync_impl(guest_t *g) } pthread_mutex_unlock(&fd_lock); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < g->nregions && n < (int) cap; i++) { const guest_region_t *r = &g->regions[i]; if (!r->shared || r->backing_fd < 0) @@ -552,7 +553,7 @@ static int64_t sc_sync_impl(guest_t *g) continue; hosts[n++] = duped; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); /* fsync each dup outside both locks so a slow disk does not stall * concurrent FD or memory operations on other threads. @@ -738,7 +739,7 @@ static int64_t sc_mincore(guest_t *g, * never early-returns on a hole. */ uint8_t chunk[512]; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int ri = guest_region_first_end_above(g, addr); for (uint64_t done = 0; done < npages;) { uint64_t batch = npages - done; @@ -753,13 +754,20 @@ static int64_t sc_mincore(guest_t *g, if (!mapped) has_hole = true; } - if (guest_write(g, vec + done, chunk, batch) < 0) { - pthread_mutex_unlock(&mmap_lock); + + /* sc_mincore holds mmap_lock while regions[] is swept. Materialize a + * valid lazy output block through the locked entry point, then use a + * no-fault copy so an invalid vec returns EFAULT instead of trying to + * acquire mmap_lock recursively. + */ + (void) guest_lazy_faultin_locked(g, vec + done, batch); + if (guest_write_nofault(g, vec + done, chunk, batch) < 0) { + mmap_lock_release(); return -LINUX_EFAULT; } done += batch; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return has_hole ? -LINUX_ENOMEM : 0; } @@ -976,6 +984,19 @@ static int64_t sc_set_tid_address(guest_t *g, return proc_get_pid(); } +static uint64_t mmap_fastpath_eligible_length(uint64_t addr, + uint64_t length, + uint64_t prot, + uint64_t flags) +{ + if (addr != 0 || prot != (LINUX_PROT_READ | LINUX_PROT_WRITE) || + (flags & ~(uint64_t) LINUX_MAP_NORESERVE) != + (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS) || + length == 0 || length > UINT64_MAX - (GUEST_PAGE_SIZE - 1)) + return 0; + return (length + GUEST_PAGE_SIZE - 1) & ~(GUEST_PAGE_SIZE - 1); +} + static int64_t sc_mmap(guest_t *g, uint64_t x0, uint64_t x1, @@ -985,9 +1006,31 @@ static int64_t sc_mmap(guest_t *g, uint64_t x5, bool verbose) { - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - pthread_mutex_unlock(&mmap_lock); + uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); + uint64_t arena_addr = 0; + if (refill_len && mmap_fastpath_allocate_current_publication_only( + g, refill_len, &arena_addr)) + return (int64_t) arena_addr; + + int materialized_fd = -1; + int prep = mmap_prepare_file(g, x0, x1, (int) x3, (int) x4, (int64_t) x5, + &materialized_fd); + if (prep < 0) + return prep; + mmap_lock_acquire(g); + int64_t r; + if (refill_len && + mmap_fastpath_allocate_current_locked(g, refill_len, &arena_addr)) { + r = (int64_t) arena_addr; + } else { + r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5, + materialized_fd); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + } + mmap_lock_release(); + if (materialized_fd >= 0) + close(materialized_fd); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) (uint64_t) r); @@ -1004,9 +1047,10 @@ static int64_t sc_mremap(guest_t *g, bool verbose) { (void) x5; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) x2, (int) x3, @@ -2191,10 +2235,7 @@ static int64_t sc_execve(guest_t *g, (void) x3; (void) x4; (void) x5; - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); - pthread_mutex_unlock(&mmap_lock); - return r; + return sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); } static int64_t sc_execveat(guest_t *g, @@ -2210,8 +2251,9 @@ static int64_t sc_execveat(guest_t *g, hv_vcpu_t vcpu = current_thread->vcpu; int dirfd = (int) x0, flags = (int) x4; - /* Resolve the target path before taking mmap_lock (path resolution may call - * fd_to_host / openat which do not need mmap_lock). + /* Resolve the target path before entering the exec transaction. Path + * resolution may call fd_to_host / openat and does not need mmap_lock; + * sys_execve takes it at the point of no return. */ uint64_t path_gva = x1; char resolved[LINUX_PATH_MAX]; @@ -2274,7 +2316,6 @@ static int64_t sc_execveat(guest_t *g, need_resolve = true; } - pthread_mutex_lock(&mmap_lock); int64_t r; if (need_resolve) { /* Use the host-resolved path directly so execveat does not copy a host @@ -2284,7 +2325,6 @@ static int64_t sc_execveat(guest_t *g, } else { r = sys_execve(vcpu, g, path_gva, x2, x3, verbose, NULL); } - pthread_mutex_unlock(&mmap_lock); return r; } diff --git a/src/syscall/sysvipc.c b/src/syscall/sysvipc.c index 4c4cab9b..791fc97d 100644 --- a/src/syscall/sysvipc.c +++ b/src/syscall/sysvipc.c @@ -255,8 +255,8 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) return gva; /* propagate mmap error */ } - /* Copy host shm content into guest memory */ - if (guest_write(g, (uint64_t) gva, host_addr, seg_size) < 0) { + (void) guest_lazy_faultin_locked(g, (uint64_t) gva, seg_size); + if (guest_write_nofault(g, (uint64_t) gva, host_addr, seg_size) < 0) { shmdt(host_addr); return -LINUX_EFAULT; } @@ -312,20 +312,16 @@ int64_t sys_shmdt(guest_t *g, uint64_t shmaddr_gva) match->active = false; pthread_mutex_unlock(&shm_lock); - /* Write back guest modifications to host shm (unless read-only) */ if (!entry.rdonly) { - /* Read guest memory back to host shm buffer */ - guest_read(g, entry.guest_gva, entry.host_addr, entry.size); + (void) guest_lazy_faultin_locked(g, entry.guest_gva, entry.size); + (void) guest_read_nofault(g, entry.guest_gva, entry.host_addr, + entry.size); } /* Detach host shm */ shmdt(entry.host_addr); - /* Guest memory remains allocated (no munmap). Linux shmdt does not - * guarantee immediate unmap either; the pages become undefined. A real - * implementation would munmap the guest region here. - */ - + /* Guest backing remains allocated; only the host attachment is released. */ return 0; } diff --git a/src/syscall/usbdev.c b/src/syscall/usbdev.c index 7529c9a0..7ab634fa 100644 --- a/src/syscall/usbdev.c +++ b/src/syscall/usbdev.c @@ -969,17 +969,14 @@ static void usbdev_unref(usbdev_t *u) * transfer. The in-code claim that this "briefly" stalled other fds' lookups * was neither brief nor bounded. */ -static usbdev_t *usbdev_acquire(int fd) +static usbdev_t *usbdev_acquire_generation(int fd, uint64_t generation) { - fd_entry_t snap; - if (!fd_snapshot(fd, &snap) || snap.type != FD_USBDEV) - return NULL; usbdev_t *u = NULL; pthread_mutex_lock(&usbdev_table_lock); for (int i = 0; i < USBDEV_MAX_FDS; i++) { if (usbdev_fds[i].used && !usbdev_fds[i].dead && usbdev_fds[i].guest_fd == fd && - usbdev_fds[i].generation == snap.generation) { + usbdev_fds[i].generation == generation) { u = &usbdev_fds[i]; u->refs++; break; @@ -997,6 +994,14 @@ static usbdev_t *usbdev_acquire(int fd) return u; } +static usbdev_t *usbdev_acquire(int fd) +{ + fd_entry_t snap; + if (!fd_snapshot(fd, &snap) || snap.type != FD_USBDEV) + return NULL; + return usbdev_acquire_generation(fd, snap.generation); +} + /* Unlock and unpin an entry usbdev_acquire returned. */ static void usbdev_release(usbdev_t *u) { @@ -1465,6 +1470,22 @@ static unsigned usbdev_fmode(int linux_flags) return (unsigned) (((linux_flags & LINUX_O_ACCMODE) + 1) & 3); } +static void usbdev_prefault_read_locked(usbdev_t *u, + guest_t *g, + uint64_t buf_gva, + uint64_t count) +{ + uint64_t len = count < u->blob_len ? count : u->blob_len; + if (!len) + return; + + /* The acquire reference survives close; recheck the blob after relocking. + */ + pthread_mutex_unlock(&u->lock); + (void) guest_lazy_faultin(g, buf_gva, len); + pthread_mutex_lock(&u->lock); +} + int64_t usbdev_read(int fd, guest_t *g, uint64_t buf_gva, uint64_t count) { fd_entry_t snap; @@ -1472,16 +1493,19 @@ int64_t usbdev_read(int fd, guest_t *g, uint64_t buf_gva, uint64_t count) return -LINUX_EBADF; if (!(usbdev_fmode(snap.linux_flags) & USBDEV_FMODE_READ)) return -LINUX_EBADF; /* vfs: read needs FMODE_READ */ - usbdev_t *u = usbdev_acquire(fd); + usbdev_t *u = usbdev_acquire_generation(fd, snap.generation); if (!u) return -LINUX_EBADF; + usbdev_prefault_read_locked(u, g, buf_gva, count); int64_t ret; - if ((uint64_t) u->pos >= u->blob_len || count == 0) { + if (!u->blob) { + ret = -LINUX_EBADF; + } else if ((uint64_t) u->pos >= u->blob_len || count == 0) { ret = 0; } else { size_t avail = u->blob_len - (size_t) u->pos; size_t n = count < avail ? (size_t) count : avail; - if (guest_write(g, buf_gva, u->blob + u->pos, n) < 0) { + if (guest_write_nofault(g, buf_gva, u->blob + u->pos, n) < 0) { ret = -LINUX_EFAULT; } else { u->pos += (off_t) n; @@ -1510,16 +1534,19 @@ int64_t usbdev_pread(int fd, return -LINUX_EBADF; if (!(usbdev_fmode(snap.linux_flags) & USBDEV_FMODE_READ)) return -LINUX_EBADF; /* vfs: read needs FMODE_READ */ - usbdev_t *u = usbdev_acquire(fd); + usbdev_t *u = usbdev_acquire_generation(fd, snap.generation); if (!u) return -LINUX_EBADF; + usbdev_prefault_read_locked(u, g, buf_gva, count); int64_t ret; - if ((uint64_t) offset >= u->blob_len || count == 0) { + if (!u->blob) { + ret = -LINUX_EBADF; + } else if ((uint64_t) offset >= u->blob_len || count == 0) { ret = 0; } else { size_t avail = u->blob_len - (size_t) offset; size_t n = count < avail ? (size_t) count : avail; - if (guest_write(g, buf_gva, u->blob + offset, n) < 0) + if (guest_write_nofault(g, buf_gva, u->blob + offset, n) < 0) ret = -LINUX_EFAULT; else ret = (int64_t) n; @@ -1610,12 +1637,10 @@ int64_t usbdev_fstat(int fd, struct stat *st) /* ioctl handlers (entry lock held unless noted) */ -static int64_t usbdev_do_control(usbdev_t *u, guest_t *g, uint64_t arg) +static int64_t usbdev_do_control(usbdev_t *u, + guest_t *g, + linux_usbdevfs_ctrltransfer_t ct) { - linux_usbdevfs_ctrltransfer_t ct; - if (guest_read_small(g, arg, &ct, sizeof(ct)) < 0) - return -LINUX_EFAULT; - /* check_ctrlrecip (devio.c:878-935) runs before the wLength cap * (devio.c:1177-1183): a request naming an interface or endpoint the device * does not have is -ENOENT however long it is. Capping first answered @@ -1651,7 +1676,8 @@ static int64_t usbdev_do_control(usbdev_t *u, guest_t *g, uint64_t arg) return -LINUX_ENOMEM; } bool in = (ct.bRequestType & 0x80) != 0; - if (!in && ct.wLength > 0 && guest_read(g, ct.data, buf, ct.wLength) < 0) { + if (!in && ct.wLength > 0 && + guest_read_nofault(g, ct.data, buf, ct.wLength) < 0) { free(buf); return -LINUX_EFAULT; } @@ -1684,7 +1710,8 @@ static int64_t usbdev_do_control(usbdev_t *u, guest_t *g, uint64_t arg) return err; } int64_t actlen = req.wLenDone; - if (in && actlen > 0 && guest_write(g, ct.data, buf, (size_t) actlen) < 0) { + if (in && actlen > 0 && + guest_write_nofault(g, ct.data, buf, (size_t) actlen) < 0) { free(buf); return -LINUX_EFAULT; } @@ -1692,12 +1719,10 @@ static int64_t usbdev_do_control(usbdev_t *u, guest_t *g, uint64_t arg) return actlen; } -static int64_t usbdev_do_bulk(usbdev_t *u, guest_t *g, uint64_t arg) +static int64_t usbdev_do_bulk(usbdev_t *u, + guest_t *g, + linux_usbdevfs_bulktransfer_t bt) { - linux_usbdevfs_bulktransfer_t bt; - if (guest_read_small(g, arg, &bt, sizeof(bt)) < 0) - return -LINUX_EFAULT; - /* do_proc_bulk resolves and claims the endpoint's interface before it looks * at the length (devio.c:1289-1298), so an absent endpoint is -ENOENT * whatever the length says. Checking the length first answered -ENOMEM and @@ -1755,12 +1780,12 @@ static int64_t usbdev_do_bulk(usbdev_t *u, guest_t *g, uint64_t arg) int64_t err = ioret_neg_errno(r); if (err < 0) { ret = err; /* partial data not copied on error, as Linux */ - } else if (size > 0 && guest_write(g, bt.data, buf, size) < 0) { + } else if (size > 0 && guest_write_nofault(g, bt.data, buf, size) < 0) { ret = -LINUX_EFAULT; } else { ret = size; } - } else if (bt.len > 0 && guest_read(g, bt.data, buf, bt.len) < 0) { + } else if (bt.len > 0 && guest_read_nofault(g, bt.data, buf, bt.len) < 0) { ret = -LINUX_EFAULT; } else { IOReturn r = (*fi->intf)->WritePipeTO(fi->intf, pipe, buf, bt.len, @@ -1807,7 +1832,7 @@ static bool usbdev_iface_claimed_elsewhere(const usbdev_t *u, unsigned ifnum) static int64_t usbdev_do_getdriver(usbdev_t *u, guest_t *g, uint64_t arg) { linux_usbdevfs_getdriver_t gd; - if (guest_read_small(g, arg, &gd, sizeof(gd)) < 0) + if (guest_read_nofault(g, arg, &gd, sizeof(gd)) < 0) return -LINUX_EFAULT; /* Ahead of everything below: an interface question about a device that is @@ -1839,7 +1864,7 @@ static int64_t usbdev_do_getdriver(usbdev_t *u, guest_t *g, uint64_t arg) if (!bound) return -LINUX_ENODATA; } - if (guest_write_small(g, arg, &gd, sizeof(gd)) < 0) + if (guest_write_nofault(g, arg, &gd, sizeof(gd)) < 0) return -LINUX_EFAULT; return 0; } @@ -1847,7 +1872,7 @@ static int64_t usbdev_do_getdriver(usbdev_t *u, guest_t *g, uint64_t arg) static int64_t usbdev_do_setinterface(usbdev_t *u, guest_t *g, uint64_t arg) { linux_usbdevfs_setinterface_t si; - if (guest_read_small(g, arg, &si, sizeof(si)) < 0) + if (guest_read_nofault(g, arg, &si, sizeof(si)) < 0) return -LINUX_EFAULT; int64_t rc = usbdev_claim_locked(u, si.interface); /* implicit claim */ if (rc < 0) @@ -1909,7 +1934,7 @@ static bool usbdev_claimed_elsewhere(usbdev_t *u) static int64_t usbdev_do_setconfiguration(usbdev_t *u, guest_t *g, uint64_t arg) { uint32_t cfg; - if (guest_read_small(g, arg, &cfg, sizeof(cfg)) < 0) + if (guest_read_nofault(g, arg, &cfg, sizeof(cfg)) < 0) return -LINUX_EFAULT; /* -1/0 -> unconfigure (message.c:2064); SetConfiguration(0) does that. */ if (cfg == 0xffffffffu) @@ -1970,7 +1995,7 @@ static int64_t usbdev_do_setconfiguration(usbdev_t *u, guest_t *g, uint64_t arg) static int64_t usbdev_do_clear_halt(usbdev_t *u, guest_t *g, uint64_t arg) { uint32_t ep; - if (guest_read_small(g, arg, &ep, sizeof(ep)) < 0) + if (guest_read_nofault(g, arg, &ep, sizeof(ep)) < 0) return -LINUX_EFAULT; usbdev_iface_t *fi; uint8_t pipe; @@ -1996,7 +2021,7 @@ static int64_t usbdev_do_resetep(usbdev_t *u, guest_t *g, uint64_t arg) static int64_t usbdev_do_disconnect_claim(usbdev_t *u, guest_t *g, uint64_t arg) { linux_usbdevfs_disconnect_claim_t dc; - if (guest_read_small(g, arg, &dc, sizeof(dc)) < 0) + if (guest_read_nofault(g, arg, &dc, sizeof(dc)) < 0) return -LINUX_EFAULT; /* proc_disconnect_claim has no range check of its own: usb_ifnum_to_if @@ -2044,7 +2069,7 @@ static int64_t usbdev_do_disconnect_claim(usbdev_t *u, guest_t *g, uint64_t arg) static int64_t usbdev_do_driver_ioctl(usbdev_t *u, guest_t *g, uint64_t arg) { linux_usbdevfs_ioctl_t ic; - if (guest_read_small(g, arg, &ic, sizeof(ic)) < 0) + if (guest_read_nofault(g, arg, &ic, sizeof(ic)) < 0) return -LINUX_EFAULT; if (ic.ifno < 0 || ic.ifno >= USBDEV_MAX_IFACES) return -LINUX_EINVAL; @@ -2120,6 +2145,31 @@ static int64_t usbdev_speed_enum(unsigned code) } } +static size_t usbdev_ioctl_arg_size(uint32_t request) +{ + switch (request) { + case USBDEVFS_CLAIMINTERFACE: + case USBDEVFS_RELEASEINTERFACE: + case USBDEVFS_SETCONFIGURATION: + case USBDEVFS_CLEAR_HALT: + case USBDEVFS_RESETEP: + case USBDEVFS_GET_CAPABILITIES: + return sizeof(uint32_t); + case USBDEVFS_SETINTERFACE: + return sizeof(linux_usbdevfs_setinterface_t); + case USBDEVFS_GETDRIVER: + return sizeof(linux_usbdevfs_getdriver_t); + case USBDEVFS_CONNECTINFO: + return sizeof(linux_usbdevfs_connectinfo_t); + case USBDEVFS_DISCONNECT_CLAIM: + return sizeof(linux_usbdevfs_disconnect_claim_t); + case USBDEVFS_IOCTL: + return sizeof(linux_usbdevfs_ioctl_t); + default: + return 0; + } +} + int64_t usbdev_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) { fd_entry_t snap; @@ -2129,7 +2179,25 @@ int64_t usbdev_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) if (!(usbdev_fmode(snap.linux_flags) & USBDEV_FMODE_WRITE)) return -LINUX_EPERM; - usbdev_t *u = usbdev_acquire(fd); + linux_usbdevfs_ctrltransfer_t ct = {0}; + linux_usbdevfs_bulktransfer_t bt = {0}; + int copy_rc = 0; + if ((uint32_t) request == USBDEVFS_CONTROL) { + copy_rc = guest_read(g, arg, &ct, sizeof(ct)); + if (copy_rc == 0 && ct.wLength && ct.wLength <= USBDEV_CTRL_MAX) + (void) guest_lazy_faultin(g, ct.data, ct.wLength); + } else if ((uint32_t) request == USBDEVFS_BULK) { + copy_rc = guest_read(g, arg, &bt, sizeof(bt)); + if (copy_rc == 0 && bt.len && + bt.len <= USBDEV_MEMORY_MAX - USBDEV_URB_OVERHEAD) + (void) guest_lazy_faultin(g, bt.data, bt.len); + } else { + size_t len = usbdev_ioctl_arg_size((uint32_t) request); + if (len) + (void) guest_lazy_faultin(g, arg, len); + } + + usbdev_t *u = usbdev_acquire_generation(fd, snap.generation); if (!u) return -LINUX_EBADF; @@ -2137,14 +2205,14 @@ int64_t usbdev_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) switch ((uint32_t) request) { case USBDEVFS_CLAIMINTERFACE: { uint32_t ifnum; - ret = guest_read_small(g, arg, &ifnum, sizeof(ifnum)) < 0 + ret = guest_read_nofault(g, arg, &ifnum, sizeof(ifnum)) < 0 ? -LINUX_EFAULT : usbdev_claim_locked(u, ifnum); break; } case USBDEVFS_RELEASEINTERFACE: { uint32_t ifnum; - ret = guest_read_small(g, arg, &ifnum, sizeof(ifnum)) < 0 + ret = guest_read_nofault(g, arg, &ifnum, sizeof(ifnum)) < 0 ? -LINUX_EFAULT : usbdev_release_locked(u, ifnum); break; @@ -2166,8 +2234,9 @@ int64_t usbdev_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) break; case USBDEVFS_GET_CAPABILITIES: { uint32_t caps = USBDEV_CAPS; - ret = guest_write_small(g, arg, &caps, sizeof(caps)) < 0 ? -LINUX_EFAULT - : 0; + ret = guest_write_nofault(g, arg, &caps, sizeof(caps)) < 0 + ? -LINUX_EFAULT + : 0; break; } case USBDEVFS_GET_SPEED: @@ -2178,15 +2247,15 @@ int64_t usbdev_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) .devnum = (uint32_t) u->devnum, .slow = u->speed_code == 0, }; - ret = - guest_write_small(g, arg, &ci, sizeof(ci)) < 0 ? -LINUX_EFAULT : 0; + ret = guest_write_nofault(g, arg, &ci, sizeof(ci)) < 0 ? -LINUX_EFAULT + : 0; break; } case USBDEVFS_CONTROL: - ret = usbdev_do_control(u, g, arg); + ret = copy_rc < 0 ? -LINUX_EFAULT : usbdev_do_control(u, g, ct); break; case USBDEVFS_BULK: - ret = usbdev_do_bulk(u, g, arg); + ret = copy_rc < 0 ? -LINUX_EFAULT : usbdev_do_bulk(u, g, bt); break; case USBDEVFS_RESET: { /* Stage-2 deviation (see file header): clear stalls on every claimed diff --git a/tests/bench-futex.c b/tests/bench-futex.c index a4c4b6ae..24481143 100644 --- a/tests/bench-futex.c +++ b/tests/bench-futex.c @@ -39,7 +39,7 @@ * is measured. * * Run under ELFUSE_SHIM_STATS=1 to attribute the fast-path rows: the host - * prints FUTEX_EAGAIN_HIT, FUTEX_EFAULT_HIT, FUTEX_SHAPE_BAIL and + * prints FUTEX_EAGAIN_HIT, FUTEX_FAULT_BAIL, FUTEX_SHAPE_BAIL and * FUTEX_MATCH_BAIL at exit. Those four sum to every SYS_futex that reached the * shape decoder; a call arriving with attention raised lands in ATTN_BAIL * instead, since the attention load runs after the decode. Each row runs iters diff --git a/tests/bench-mmap-fresh b/tests/bench-mmap-fresh new file mode 100755 index 00000000..11cc280d --- /dev/null +++ b/tests/bench-mmap-fresh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Accumulate auxiliary fresh bump-tail mmap timing across independent elfuse +# processes. One process cannot retain enough large fresh mappings to overcome +# CNTVCT's 41.7-ns tick without exhausting guest VA. Each invocation below gets +# a new guest, while its in-guest timer excludes process startup from the +# measurement. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" + +# 1024 ticks = about 42.7 us on Apple Silicon. This bounds the aggregate counter +# quantization to roughly 0.1%, while keeping the largest cases timely. +TARGET_TICKS="${BENCH_MMAP_FRESH_TARGET_TICKS:-1024}" +MAX_RUNS="${BENCH_MMAP_FRESH_MAX_RUNS:-8192}" + +# size in bytes : mappings per fresh guest run +CASES=( + "4096:1000" + "65536:1000" + "1048576:1000" + "2097152:1000" + "8388608:256" + "134217728:16" + "1073741824:8" + "8589934592:4" + "34359738368:2" +) + +# Optional positional cases use the same size:count spelling as CASES, e.g. +# tests/bench-mmap-fresh 1073741824:8 8589934592:4 +if [ "$#" -gt 0 ]; then + CASES=("$@") +fi + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi + +printf 'Auxiliary fresh bump-tail mmap: target %s aggregate CNTVCT ticks\n' \ + "$TARGET_TICKS" +printf '%-10s %8s %6s %14s\n' size count runs 'fresh mmap ns' + +for case in "${CASES[@]}"; do + size="${case%%:*}" + count="${case##*:}" + total_ticks=0 + total_ops=0 + runs=0 + ns_per_tick= + failed=0 + + while awk -v ticks="$total_ticks" -v target="$TARGET_TICKS" \ + 'BEGIN { exit !(ticks < target) }'; do + if [ "$runs" -ge "$MAX_RUNS" ]; then + echo "fresh benchmark exceeded ${MAX_RUNS} runs for ${size}" >&2 + failed=1 + break + fi + if ! output="$("$ELFUSE" "$BENCH" fresh "$size" "$count" 2>&1)"; then + echo "fresh guest run failed for size=${size}, count=${count}:" >&2 + echo "$output" >&2 + failed=1 + break + fi + raw_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ticks=/) {sub(/^ticks=/, "", $i); print $i}}' <<< "$output")" + read_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^read_ticks=/) {sub(/^read_ticks=/, "", $i); print $i}}' <<< "$output")" + ns_per_tick="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ns_per_tick=/) {sub(/^ns_per_tick=/, "", $i); print $i}}' <<< "$output")" + if [ -z "$raw_ticks" ] || [ -z "$read_ticks" ] || [ -z "$ns_per_tick" ]; then + echo "unexpected benchmark output: $output" >&2 + failed=1 + break + fi + total_ticks="$(awk -v total="$total_ticks" -v raw="$raw_ticks" -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + total_ops=$((total_ops + count)) + runs=$((runs + 1)) + done + + human="$(awk -v n="$size" 'BEGIN { if (n >= 1073741824) printf "%g GiB", n / 1073741824; else if (n >= 1048576) printf "%g MiB", n / 1048576; else printf "%g KiB", n / 1024 }')" + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" FAILED + continue + fi + ns_per_op="$(awk -v ticks="$total_ticks" -v ops="$total_ops" -v ns="$ns_per_tick" 'BEGIN { printf "%.2f", ticks * ns / ops }')" + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" "$ns_per_op" +done diff --git a/tests/bench-mmap-isolated b/tests/bench-mmap-isolated new file mode 100755 index 00000000..726fa881 --- /dev/null +++ b/tests/bench-mmap-isolated @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# mmap/munmap performance tests isolated across elfuse processes. +# +# Section A launches a brand new elfuse process for every timed mmap/munmap +# pair. Section B takes an outer median across fresh processes for each size. +# Section C takes an outer distribution of warmed samples across fresh processes +# for each size. Section D pairs an empty VM exit with the next exit that drains +# EL1 fast-path work. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=tests/lib/bench-mmap.sh +. "$ROOT_DIR/tests/lib/bench-mmap.sh" + +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" + +# One timed call is comparable to CNTVCT's 41.7-ns period. Accumulate corrected +# ticks across fresh processes before converting them to a per-call result. +ONESHOT_TARGET_TICKS="${BENCH_MMAP_ONESHOT_TARGET_TICKS:-256}" +ONESHOT_MAX_RUNS="${BENCH_MMAP_ONESHOT_MAX_RUNS:-4096}" +MATERIALIZED_GUESTS="${BENCH_MMAP_MATERIALIZED_GUESTS:-15}" +DIRTY_GUESTS="${BENCH_MMAP_DIRTY_GUESTS:-41}" +HOST_DRAIN_SAMPLES="${BENCH_MMAP_HOST_DRAIN_SAMPLES:-31}" +SIZE_POINTS="${BENCH_MMAP_SIZE_POINTS:-41}" + +log_sizes() +{ + awk -v min="$1" -v max="$2" -v count="$3" 'BEGIN { + page = 4096 + last = 0 + span = log(max) - log(min) + for (i = 0; i < count; i++) { + if (i == 0) + value = min + else if (i == count - 1) + value = max + else { + exact = exp(log(min) + span * i / (count - 1)) + value = int((exact + page - 1) / page) * page + if (value <= last) + value = last + page + } + printf "%.0f\n", value + last = value + } + }' +} + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi +if ! [[ "$MATERIALIZED_GUESTS" =~ ^[1-9][0-9]*$ ]]; then + echo "BENCH_MMAP_MATERIALIZED_GUESTS must be a positive integer" >&2 + exit 2 +fi +if ! [[ "$DIRTY_GUESTS" =~ ^[1-9][0-9]*$ ]] || + [ "$DIRTY_GUESTS" -lt 41 ] || [ "$DIRTY_GUESTS" -gt 100000 ]; then + echo "BENCH_MMAP_DIRTY_GUESTS must be in 41..100000" >&2 + exit 2 +fi +if ! [[ "$HOST_DRAIN_SAMPLES" =~ ^[1-9][0-9]*$ ]] || + [ "$HOST_DRAIN_SAMPLES" -gt 100000 ]; then + echo "BENCH_MMAP_HOST_DRAIN_SAMPLES must be in 1..100000" >&2 + exit 2 +fi +if ! [[ "$SIZE_POINTS" =~ ^[0-9]+$ ]] || [ "$SIZE_POINTS" -lt 35 ] || + [ "$SIZE_POINTS" -gt 45 ]; then + echo "BENCH_MMAP_SIZE_POINTS must be in 35..45" >&2 + exit 2 +fi + +# A and B span 4 KiB to 32 GiB. C retains its 4 KiB to 1 GiB bounds. +SIZES=() +while IFS= read -r size; do + SIZES+=("$size") +done < <(log_sizes 4096 34359738368 "$SIZE_POINTS") + +DIRTY_SIZES=() +while IFS= read -r size; do + DIRTY_SIZES+=("$size") +done < <(log_sizes 4096 1073741824 "$SIZE_POINTS") + +numeric_summary() +{ + printf '%s\n' "$@" | LC_ALL=C sort -n | awk '{ values[NR] = $1 } + END { + p50 = (NR - 1) * 0.50 + p95 = (NR - 1) * 0.95 + p50lo = int(p50) + 1 + p95lo = int(p95) + 1 + p50hi = p50lo + (p50lo < NR) + p95hi = p95lo + (p95lo < NR) + p50v = values[p50lo] + (values[p50hi] - values[p50lo]) * \ + (p50 - int(p50)) + p95v = values[p95lo] + (values[p95hi] - values[p95lo]) * \ + (p95 - int(p95)) + print p50v, p95v, values[NR] + }' +} + +run_a() +{ + printf 'A. one timed mmap / munmap fast-path pair per fresh elfuse process\n' + printf ' aggregate target: %s corrected CNTVCT ticks per call type\n' \ + "$ONESHOT_TARGET_TICKS" + printf '%-10s %8s %12s %12s\n' size guests 'mmap ns' 'munmap ns' + for size in "${SIZES[@]}"; do + mmap_ticks=0 + munmap_ticks=0 + runs=0 + ns_per_tick= + failed=0 + while awk -v mmap_ticks="$mmap_ticks" -v munmap_ticks="$munmap_ticks" \ + -v target="$ONESHOT_TARGET_TICKS" \ + 'BEGIN { exit ! (mmap_ticks < target || munmap_ticks < target) }'; do + if [ "$runs" -ge "$ONESHOT_MAX_RUNS" ]; then + printf 'one-shot benchmark exceeded %s runs for %s\n' \ + "$ONESHOT_MAX_RUNS" "$size" >&2 + return 1 + fi + if ! out="$("$ELFUSE" "$BENCH" a-one "$size" 2>&1)"; then + failed=1 + break + fi + raw_mmap_ticks="$(field "$out" mmap_ticks)" + raw_munmap_ticks="$(field "$out" munmap_ticks)" + read_ticks="$(field "$out" read_ticks)" + ns_per_tick="$(field "$out" ns_per_tick)" + mmap_ticks="$(awk -v total="$mmap_ticks" -v raw="$raw_mmap_ticks" \ + -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + munmap_ticks="$(awk -v total="$munmap_ticks" \ + -v raw="$raw_munmap_ticks" -v read="$read_ticks" \ + 'BEGIN { print total + raw - read }')" + runs=$((runs + 1)) + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %12s %12s\n' "$(human "$size")" - FAILED - + continue + fi + mmap_ns="$(awk -v ticks="$mmap_ticks" -v runs="$runs" \ + -v ns="$ns_per_tick" 'BEGIN { print ticks * ns / runs }')" + munmap_ns="$(awk -v ticks="$munmap_ticks" -v runs="$runs" \ + -v ns="$ns_per_tick" 'BEGIN { print ticks * ns / runs }')" + printf '%-10s %8s %12.1f %12.1f\n' "$(human "$size")" "$runs" \ + "$mmap_ns" "$munmap_ns" + done + printf '\n' +} + +run_b() +{ + printf 'B. munmap after materializing one 4 KiB page (isolated)\n' + printf ' outer median: %s fresh elfuse processes per size\n' \ + "$MATERIALIZED_GUESTS" + printf '%-10s %8s %12s\n' size guests 'munmap ns' + for size in "${SIZES[@]}"; do + munmap_samples=() + failed=0 + for ((guest = 0; guest < MATERIALIZED_GUESTS; guest++)); do + if ! out="$("$ELFUSE" "$BENCH" b-one "$size" 2>&1)"; then + failed=1 + break + fi + munmap_samples+=("$(field "$out" munmap_ns)") + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %12s\n' "$(human "$size")" - FAILED + continue + fi + munmap_ns="$(numeric_median "${munmap_samples[@]}")" + printf '%-10s %8u %12.1f\n' "$(human "$size")" \ + "$MATERIALIZED_GUESTS" "$munmap_ns" + done + printf '\n' +} + +run_c() +{ + printf 'C. munmap after dirtying every 4 KiB page (isolated)\n' + printf ' %s fresh processes: one warmup, one timed sample each\n' \ + "$DIRTY_GUESTS" + printf '%-10s %8s %12s %12s %12s\n' size guests 'p50 ns' 'p95 ns' 'max ns' + for size in "${DIRTY_SIZES[@]}"; do + munmap_samples=() + failed=0 + for ((guest = 0; guest < DIRTY_GUESTS; guest++)); do + if ! out="$("$ELFUSE" "$BENCH" c-one "$size" 2>&1)"; then + failed=1 + break + fi + munmap_samples+=("$(field "$out" munmap_ns)") + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %12s %12s %12s\n' "$(human "$size")" - \ + FAILED - - + continue + fi + read -r p50_ns p95_ns max_ns \ + < <(numeric_summary "${munmap_samples[@]}") + printf '%-10s %8s %12.1f %12.1f %12.1f\n' "$(human "$size")" \ + "$DIRTY_GUESTS" "$p50_ns" "$p95_ns" "$max_ns" + done + printf '\n' +} + +run_d() +{ + "$ELFUSE" "$BENCH" host-drain "$HOST_DRAIN_SAMPLES" +} + +case "${1:-all}" in + a) run_a ;; + b) run_b ;; + c) run_c ;; + d) run_d ;; + all) run_a; run_b; run_c; run_d ;; + *) + echo "usage: $0 [a|b|c|d|all]" >&2 + exit 2 + ;; +esac diff --git a/tests/bench-mmap-lazy.c b/tests/bench-mmap-lazy.c new file mode 100644 index 00000000..e6545672 --- /dev/null +++ b/tests/bench-mmap-lazy.c @@ -0,0 +1,128 @@ +/* + * Guest microbenchmark for anonymous private mmap latency vs size. + * + * Measures mmap(), first-touch, and munmap() latency for MAP_PRIVATE | + * MAP_ANONYMOUS mappings from 4 KiB to 32 GiB. A lazy (deferred page-table) + * implementation should show size-independent mmap/munmap cost; an eager + * implementation scales linearly with length and exhausts resources on the + * multi-GiB sizes. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void bench_size(uint64_t size, int iters) +{ + uint64_t t_map = 0, t_touch = 0, t_unmap = 0; + int ok = 0; + + for (int i = 0; i < iters; i++) { + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = now_ns(); + if (p == MAP_FAILED) { + printf("%10llu KiB: mmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + /* First touch: one write at the start and one mid-mapping. */ + volatile char *c = p; + c[0] = 1; + c[size / 2] = 1; + uint64_t t2 = now_ns(); + int rc = munmap(p, size); + uint64_t t3 = now_ns(); + if (rc != 0) { + printf("%10llu KiB: munmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + t_map += t1 - t0; + t_touch += t2 - t1; + t_unmap += t3 - t2; + ok++; + } + printf( + "%10llu KiB: mmap %10llu ns touch2 %10llu ns munmap %10llu ns " + "(%d iters)\n", + (unsigned long long) (size >> 10), + (unsigned long long) (t_map / (uint64_t) ok), + (unsigned long long) (t_touch / (uint64_t) ok), + (unsigned long long) (t_unmap / (uint64_t) ok), ok); +} + +int main(int argc, char **argv) +{ + static const struct { + uint64_t size; + int iters; + } cases[] = { + {4ull << 10, 200}, {64ull << 10, 200}, {2ull << 20, 100}, + {64ull << 20, 20}, {512ull << 20, 10}, {2ull << 30, 5}, + {8ull << 30, 3}, {16ull << 30, 3}, {32ull << 30, 3}, + {64ull << 30, 1}, {128ull << 30, 1}, {256ull << 30, 1}, + }; + + /* Optional argv[1]: cap size in GiB (eager implementations commit host + * memory for every byte mapped; the full matrix would thrash small hosts). + */ + uint64_t cap = ~0ull; + if (argc > 2) { + fprintf(stderr, "usage: %s [positive-cap-GiB]\n", argv[0]); + return 2; + } + if (argc == 2) { + char *end; + errno = 0; + unsigned long long gib = strtoull(argv[1], &end, 10); + if (argv[1][0] < '0' || argv[1][0] > '9' || *end != '\0' || + errno == ERANGE || gib == 0 || gib > (UINT64_MAX >> 30)) { + fprintf(stderr, + "cap must be a positive integer GiB value no greater than " + "%llu\n", + (unsigned long long) (UINT64_MAX >> 30)); + return 2; + } + cap = (uint64_t) gib << 30; + } + + setvbuf(stdout, NULL, _IONBF, 0); + printf("anon private mmap latency vs size\n"); + for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + if (cases[i].size <= cap) + bench_size(cases[i].size, cases[i].iters); + + /* Full-touch throughput sanity: 64 MiB written end to end. */ + uint64_t size = 64ull << 20; + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf("full-touch mmap failed: %s\n", strerror(errno)); + return 1; + } + memset(p, 0xa5, size); + uint64_t t1 = now_ns(); + munmap(p, size); + printf("mmap+memset 64MiB: %llu ns (%.2f GiB/s)\n", + (unsigned long long) (t1 - t0), + (double) size / 1.073741824 / (double) (t1 - t0)); + return 0; +} diff --git a/tests/bench-mmap-mt b/tests/bench-mmap-mt new file mode 100755 index 00000000..dde8e237 --- /dev/null +++ b/tests/bench-mmap-mt @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Measure concurrent mmap and munmap fast paths in fresh elfuse processes. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=tests/lib/bench-mmap.sh +. "$ROOT_DIR/tests/lib/bench-mmap.sh" + +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" +SAMPLES="${BENCH_MMAP_MT_SAMPLES:-7}" + +SIZES=(4096 2097152) +THREAD_COUNTS=(1 2 4 8) + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi +if ! [[ "$SAMPLES" =~ ^[1-9][0-9]*$ ]]; then + echo "BENCH_MMAP_MT_SAMPLES must be a positive integer" >&2 + exit 2 +fi + +printf 'Multi-threaded mmap/munmap fast paths\n' +printf 'Each row is the median of %s fresh elfuse processes.\n' "$SAMPLES" +printf 'ns is mean per-thread time; Mops/s is aggregate throughput.\n' +printf '%-10s %8s %8s %12s %12s %12s %12s\n' \ + size threads ops/thr 'mmap ns' 'mmap Mops/s' 'munmap ns' 'munmap Mops/s' + +for size in "${SIZES[@]}"; do + for threads in "${THREAD_COUNTS[@]}"; do + mmap_ns_samples=() + mmap_mops_samples=() + munmap_ns_samples=() + munmap_mops_samples=() + ops_per_thread=- + failed=0 + for ((sample = 0; sample < SAMPLES; sample++)); do + if ! out="$("$ELFUSE" "$BENCH" mt-one "$size" "$threads" 2>&1)"; then + printf 'benchmark failed for size=%s threads=%s:\n%s\n' \ + "$size" "$threads" "$out" >&2 + failed=1 + break + fi + if ! ops_per_thread="$(field "$out" ops_per_thread)" \ + || ! mmap_ns="$(field "$out" mmap_ns)" \ + || ! mmap_mops="$(field "$out" mmap_mops)" \ + || ! munmap_ns="$(field "$out" munmap_ns)" \ + || ! munmap_mops="$(field "$out" munmap_mops)"; then + failed=1 + break + fi + mmap_ns_samples+=("$mmap_ns") + mmap_mops_samples+=("$mmap_mops") + munmap_ns_samples+=("$munmap_ns") + munmap_mops_samples+=("$munmap_mops") + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %8s %12s\n' "$(human "$size")" "$threads" - FAILED + continue + fi + mmap_ns="$(numeric_median "${mmap_ns_samples[@]}")" + mmap_mops="$(numeric_median "${mmap_mops_samples[@]}")" + munmap_ns="$(numeric_median "${munmap_ns_samples[@]}")" + munmap_mops="$(numeric_median "${munmap_mops_samples[@]}")" + printf '%-10s %8s %8s %12.1f %12.3f %12.1f %12.3f\n' \ + "$(human "$size")" "$threads" "$ops_per_thread" "$mmap_ns" \ + "$mmap_mops" "$munmap_ns" "$munmap_mops" + done +done diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c new file mode 100644 index 00000000..9b092e3b --- /dev/null +++ b/tests/bench-mmap.c @@ -0,0 +1,1069 @@ +/* + * Comprehensive anonymous-mmap microbenchmark for elfuse. + * + * Measures the guest-visible cost of the mmap subsystem in isolation: + * allocation, teardown, first-touch faults, permission splitting, and remap. It + * is self-contained, needs no external harness, and is meant to be run under + * elfuse (./build/elfuse ./build/bench-mmap) but also runs on any aarch64-linux + * host for a ground-truth comparison. + * + * Timing: reads CNTVCT_EL0 directly at EL0 (enabled by CNTKCTL_EL1.EL0VCTEN in + * bootstrap.c), so a measurement costs an isb + mrs, not a clock_gettime SVC. + * On Apple Silicon CNTFRQ is ~24 MHz (~41.7 ns/tick); amortizing over an + * adaptive batch drives the effective resolution well below one tick. This is + * the key fairness property: clock_gettime on a static guest falls through to + * the ~2 us SVC path and swamps any sub-us operation. + * + * Every in-process case takes one untimed warmup pass (to pay the one-time + * arena carve and page-table extension). Most sections report aggregate + * samples; section C retains every operation so its normal latency and long + * tail remain visible. The isolated driver gives every timed section-A and + * section-C operation a new elfuse process. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* CNTVCT timing */ + +static double ns_per_tick; + +static inline uint64_t rd(void) +{ + uint64_t v; + __asm__ volatile("isb\n\tmrs %0, cntvct_el0" : "=r"(v)); + return v; +} + +static void clock_init(void) +{ + uint64_t f; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(f)); + if (f == 0) + f = 24000000; /* defensive: assume 24 MHz if RES0 */ + ns_per_tick = 1e9 / (double) f; +} + +static double ns(uint64_t ticks) +{ + return (double) ticks * ns_per_tick; +} + +static int cmp_d(const void *a, const void *b) +{ + double x = *(const double *) a, y = *(const double *) b; + return (x > y) - (x < y); +} + +static double median(double *v, int n) +{ + qsort(v, n, sizeof(*v), cmp_d); + return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +/* R-7/sample quantile, matching the common (n - 1) * p interpolation. The input + * must already be sorted. + */ +static double sorted_quantile(const double *v, unsigned n, double p) +{ + double pos = (double) (n - 1) * p; + unsigned lo = (unsigned) pos; + unsigned hi = lo + (lo + 1 < n); + return v[lo] + (v[hi] - v[lo]) * (pos - lo); +} + +#define ITERS 15 +#define MIN_TIMED_TICKS 4096 +#define KIB (1ULL << 10) +#define MIB (1ULL << 20) +#define GIB (1ULL << 30) +#define DIRTY_VMEXIT_STRIDE (8 * MIB) + +/* Calibrate the rd()/rd() interval around a timed operation. Its median cost is + * subtracted from aggregate samples so the counter-read overhead is not + * attributed to mmap or munmap. + */ +static double rd_pair_ticks(void) +{ + enum { CAL_SAMPLES = 15, CAL_OPS = 4096 }; + double samples[CAL_SAMPLES]; + for (int sample = 0; sample < CAL_SAMPLES; sample++) { + uint64_t total = 0; + for (int op = 0; op < CAL_OPS; op++) { + uint64_t t0 = rd(); + uint64_t t1 = rd(); + total += t1 - t0; + } + samples[sample] = (double) total / CAL_OPS; + } + return median(samples, CAL_SAMPLES); +} + +static const char *human(uint64_t s, char *buf) +{ + if (s >= GIB) + sprintf(buf, "%llu GiB", (unsigned long long) (s / GIB)); + else if (s >= MIB) + sprintf(buf, "%llu MiB", (unsigned long long) (s / MIB)); + else + sprintf(buf, "%llu KiB", (unsigned long long) (s / KIB)); + return buf; +} + +/* A. One mmap and one munmap fast-path sample in a fresh guest. The host-side + * driver starts a new elfuse process for every invocation of this function. + * Prime the requested arena size outside the timed interval, then force a host + * drain so the arena is empty and its cursor is rewound. The two measured calls + * can then take the EL1 paths even when size exceeds the initial 64 MiB arena. + * Raw ticks are returned because one fast call is comparable to the counter + * period; the driver aggregates independent one-call samples before converting + * them to nanoseconds. + */ +static int bench_fastpath_once(uint64_t size) +{ + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED || munmap(warmup, size) != 0) + return 1; + + /* Drain the warmup retirement and rewind the now-empty arena. */ + (void) fcntl(-1, F_GETFD); + + uint64_t mmap_start = rd(); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t mmap_end = rd(); + if (ptr == MAP_FAILED) + return 1; + + uint64_t munmap_start = rd(); + int rc = munmap(ptr, size); + uint64_t munmap_end = rd(); + if (rc != 0) + return 1; + + printf( + "fast-once size=%llu mmap_ticks=%llu munmap_ticks=%llu " + "read_ticks=%.6f ns_per_tick=%.12f\n", + (unsigned long long) size, (unsigned long long) (mmap_end - mmap_start), + (unsigned long long) (munmap_end - munmap_start), rd_pair_ticks(), + ns_per_tick); + return 0; +} + +#define HOST_DRAIN_MAX_BATCH 30 +#define HOST_DRAIN_DEFAULT_SAMPLES 31 + +typedef struct { + double baseline_ns; + double mmap_extra_ns; + double mmap_ns_per_op; + double munmap_extra_ns; + double munmap_ns_per_op; +} host_drain_result_t; + +static bool timed_empty_vmexit(uint64_t *ticks) +{ + errno = 0; + uint64_t start = rd(); + int rc = fcntl(-1, F_GETFD); + uint64_t end = rd(); + if (rc != -1 || errno != EBADF) + return false; + *ticks = end - start; + return true; +} + +/* Pair a work-bearing fcntl VM exit with an empty one at the same mapping + * population. The difference leaves only the host's publication or retirement + * drain. Batches stop below both 32-entry ring limits so no mmap or munmap in + * the setup triggers an early HVC drain. + */ +static bool bench_host_drain_measure(uint64_t size, + unsigned batch, + unsigned samples, + host_drain_result_t *result) +{ + if (batch == 0 || batch > HOST_DRAIN_MAX_BATCH || samples == 0) + return false; + + void **mappings = calloc(batch, sizeof(*mappings)); + double *baseline = malloc(2 * samples * sizeof(*baseline)); + double *mmap_extra = malloc(samples * sizeof(*mmap_extra)); + double *munmap_extra = malloc(samples * sizeof(*munmap_extra)); + if (!mappings || !baseline || !mmap_extra || !munmap_extra) + goto fail; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t discard; + if (warmup == MAP_FAILED || munmap(warmup, size) != 0 || + !timed_empty_vmexit(&discard)) + goto fail; + + for (unsigned sample = 0; sample < samples; sample++) { + uint64_t mmap_base, mmap_work, munmap_base, munmap_work; + if (!timed_empty_vmexit(&mmap_base)) + goto fail; + + for (unsigned op = 0; op < batch; op++) { + mappings[op] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mappings[op] == MAP_FAILED) + goto fail; + } + if (!timed_empty_vmexit(&mmap_work) || + !timed_empty_vmexit(&munmap_base)) + goto fail; + + for (unsigned op = 0; op < batch; op++) { + if (munmap(mappings[op], size) != 0) + goto fail; + mappings[op] = NULL; + } + if (!timed_empty_vmexit(&munmap_work)) + goto fail; + + baseline[2 * sample] = (double) mmap_base; + baseline[2 * sample + 1] = (double) munmap_base; + mmap_extra[sample] = (double) mmap_work - (double) mmap_base; + munmap_extra[sample] = (double) munmap_work - (double) munmap_base; + } + + result->baseline_ns = median(baseline, (int) (2 * samples)) * ns_per_tick; + result->mmap_extra_ns = median(mmap_extra, (int) samples) * ns_per_tick; + result->mmap_ns_per_op = result->mmap_extra_ns / batch; + result->munmap_extra_ns = median(munmap_extra, (int) samples) * ns_per_tick; + result->munmap_ns_per_op = result->munmap_extra_ns / batch; + free(munmap_extra); + free(mmap_extra); + free(baseline); + free(mappings); + return true; + +fail: + if (mappings) { + for (unsigned op = 0; op < batch; op++) { + if (mappings[op] && mappings[op] != MAP_FAILED) + munmap(mappings[op], size); + } + (void) fcntl(-1, F_GETFD); + } + free(munmap_extra); + free(mmap_extra); + free(baseline); + free(mappings); + return false; +} + +static bool bench_host_drain(unsigned samples) +{ + static const unsigned batches[] = {1, 2, 4, 8, 16, 30}; + const uint64_t size = 4 * KIB; + bool ok = true; + + printf("== D. host work drained at the next VM exit ==\n"); + printf("Empty fcntl VM exits provide the paired fixed-cost baseline.\n"); + printf("%-10s %8s %8s %12s %14s %12s %14s %12s\n", "size", "batch", + "samples", "baseline ns", "mmap extra ns", "mmap ns/op", + "munmap extra ns", "munmap ns/op"); + for (unsigned i = 0; i < sizeof(batches) / sizeof(batches[0]); i++) { + host_drain_result_t result; + char hb[16]; + if (!bench_host_drain_measure(size, batches[i], samples, &result)) { + printf("%-10s %8u %8u %14s\n", human(size, hb), batches[i], samples, + "FAILED"); + ok = false; + continue; + } + printf("%-10s %8u %8u %12.1f %14.1f %12.1f %14.1f %12.1f\n", + human(size, hb), batches[i], samples, result.baseline_ns, + result.mmap_extra_ns, result.mmap_ns_per_op, + result.munmap_extra_ns, result.munmap_ns_per_op); + } + printf("\n"); + return ok; +} + +/* Auxiliary: fresh bump-tail mmap isolates the lazy_fresh_range path. Allocate + * sequential run WITHOUT freeing, so every mapping lands at or above the arena + * high-water, which is the case lazy_fresh_range skips the stale-PTE scan for. + * Small mappings use the original 2-GiB footprint cap; large mappings use a + * minimum count chosen to retain multiple samples without exceeding 64 GiB of + * live fresh VA. Run this binary against an opt-off build to read the skip's + * contribution as the difference on this identical code path. A MAP_FIXED + * "recycled" compare would instead measure the region-snapshot replacement + * path, not the skip. + */ +static void bench_fresh(void) +{ + static const uint64_t sizes[] = {4 * KIB, 64 * KIB, MIB, + 2 * MIB, 8 * MIB, 128 * MIB, + GIB, 8 * GIB, 32 * GIB}; + printf( + "== Auxiliary: fresh bump-tail mmap, per-mmap ns " + "(lazy_fresh_range path) ==\n"); + printf("%-10s %8s %14s\n", "size", "count", "fresh mmap ns"); + void *run[1000]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int n = (int) (2 * GIB / size); + if (n < 1) + n = 1; + if (n > 1000) + n = 1000; + if (size == GIB) + n = 8; + else if (size == 8 * GIB) + n = 4; + else if (size == 32 * GIB) + n = 2; + /* warmup one fresh mapping so the arena high-water is already primed */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + char hb[16]; + if (failed) { + printf("%-10s %8d %14s (%d failed)\n", human(size, hb), n, + "PARTIAL", failed); + continue; + } + printf("%-10s %8d %14.1f\n", human(size, hb), n, ns(t1 - t0) / n); + } + printf("\n"); +} + +/* One fresh bump-tail run for the host-side driver. A new elfuse process is + * used for each invocation, so the driver can accumulate many counter ticks + * without exhausting one guest's VA space. + */ +static int bench_fresh_one(uint64_t size, int n) +{ + void *run[1000]; + if (n < 1 || n > (int) (sizeof(run) / sizeof(run[0]))) + return 2; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED) + return 1; + munmap(warmup, size); + + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + if (failed) + return 1; + + printf( + "fresh size=%llu count=%d ticks=%llu read_ticks=%.6f " + "ns_per_tick=%.12f\n", + (unsigned long long) size, n, (unsigned long long) (t1 - t0), + rd_pair_ticks(), ns_per_tick); + return 0; +} + +/* Auxiliary: first-touch cost. Touch one byte per macOS 16 KiB host page. Each + * demands a distinct physical backing page, but it is not necessarily a + * distinct HVC: elfuse installs Stage-1 descriptors in 2 MiB windows and the + * fault-around policy may install several windows per exit. Report both the + * whole sweep and its per-host-page amortization; calling the latter a + * "per-fault" cost would substantially overcount guest translation faults. + */ +static void bench_fault(int pages, int drain_between) +{ + const uint64_t stride = 16 * KIB; + uint64_t size = stride * (uint64_t) (pages + 1); + printf( + "== Auxiliary: first-touch fault cost (16 KiB stride, %d pages%s) " + "==\n", + pages, drain_between ? ", forced retire drain" : ""); + double sweep[ITERS]; + for (int it = -1; it < ITERS; it++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED: %s\n\n", strerror(errno)); + return; + } + uint64_t t0 = rd(); + for (int i = 0; i < pages; i++) + p[(uint64_t) i * stride] = 1; + uint64_t t1 = rd(); + munmap((void *) p, size); + if (drain_between) + (void) fcntl(-1, F_GETFD); + if (it >= 0) + sweep[it] = ns(t1 - t0); + } + qsort(sweep, ITERS, sizeof(*sweep), cmp_d); + printf(" sweep: p50 %.1f us p95 %.1f us max %.1f us\n", + sorted_quantile(sweep, ITERS, 0.50) / 1000.0, + sorted_quantile(sweep, ITERS, 0.95) / 1000.0, + sweep[ITERS - 1] / 1000.0); + printf(" amortized/touch: p50 %.1f ns p95 %.1f ns max %.1f ns\n\n", + sorted_quantile(sweep, ITERS, 0.50) / pages, + sorted_quantile(sweep, ITERS, 0.95) / pages, + sweep[ITERS - 1] / pages); +} + +/* Auxiliary: mprotect split cost. Flip the middle 4 KiB of a 2 MiB RW block to + * PROT_READ, forcing guest_split_block to convert the L2 block into 512 L3 + * pages. Restore between iterations so each run does a fresh split. + */ +static void bench_mprotect_split(void) +{ + printf( + "== Auxiliary: mprotect split " + "(2 MiB block -> L3, protect middle 4 KiB) ==\n"); + double sp[ITERS]; + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint8_t *mid = p + MIB; + uint64_t t0 = rd(); + int rc = mprotect(mid, 4 * KIB, PROT_READ); + uint64_t t1 = rd(); + munmap(p, 2 * MIB); + if (rc != 0) { + printf(" mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + sp[it] = ns(t1 - t0); + } + printf(" split: median %.1f ns min %.1f ns\n\n", median(sp, ITERS), + sp[0]); +} + +/* Auxiliary: mremap grow, in-place vs forced move. */ +static void bench_mremap(void) +{ + printf("== Auxiliary: mremap grow 4 KiB -> 8 KiB ==\n"); + double inp[ITERS], mov[ITERS]; + + /* In-place: no blocker, the following page is free. */ + for (int it = -1; it < ITERS; it++) { + void *p = mmap(NULL, 4 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + munmap(p, 4 * KIB); + printf(" mremap in-place FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (it >= 0) + inp[it] = ns(t1 - t0); + } + + /* Forced move: a PROT_READ blocker sits immediately after, so the grow must + * relocate. + */ + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 8 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + + /* free the tail page and pin it read-only so in-place growth is blocked + * but the head is still a 4 KiB mapping. + */ + munmap(p + 4 * KIB, 4 * KIB); + void *blk = mmap(p + 4 * KIB, 4 * KIB, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + printf(" mremap move FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (blk != MAP_FAILED) + munmap(blk, 4 * KIB); + if (it >= 0) + mov[it] = ns(t1 - t0); + } + printf(" in-place: median %.1f ns min %.1f ns\n", median(inp, ITERS), + inp[0]); + printf(" move: median %.1f ns min %.1f ns\n\n", median(mov, ITERS), + mov[0]); +} + +/* Auxiliary: multi-threaded mmap and munmap fast paths. Each worker primes its + * arena before the start barrier, allocates untouched mappings, then retires + * them after a second barrier. The host-side driver runs each configuration in + * a fresh elfuse process. + */ +typedef struct { + uint64_t size; + int n; + void **buf; + uint64_t mmap_start; + uint64_t mmap_end; + uint64_t munmap_start; + uint64_t munmap_end; + double mmap_ns; + double munmap_ns; + int failed; +} mt_arg_t; + +typedef struct { + double mmap_ns; + double mmap_mops; + double munmap_ns; + double munmap_mops; +} mt_result_t; + +static pthread_barrier_t mt_barrier; +static pthread_mutex_t mt_start_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t mt_start_cond = PTHREAD_COND_INITIALIZER; +static int mt_start; +static int mt_abort; + +/* CNTVCT_EL0 reads a constant on worker vCPUs (EL0VCTEN is set for the main + * vCPU only), so the MT worker brackets its whole loop with clock_gettime and + * amortizes each SVC pair over n operations. + */ +static uint64_t mono_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void *mt_worker(void *p) +{ + mt_arg_t *a = p; + pthread_mutex_lock(&mt_start_lock); + while (!mt_start) + pthread_cond_wait(&mt_start_cond, &mt_start_lock); + int run = !mt_abort; + pthread_mutex_unlock(&mt_start_lock); + if (!run) + return NULL; + + void *warmup = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED || munmap(warmup, a->size) != 0) + a->failed++; + (void) fcntl(-1, F_GETFD); + + pthread_barrier_wait(&mt_barrier); + a->mmap_start = mono_ns(); + for (int i = 0; i < a->n; i++) + a->buf[i] = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + a->mmap_end = mono_ns(); + a->mmap_ns = (double) (a->mmap_end - a->mmap_start) / a->n; + + pthread_barrier_wait(&mt_barrier); + a->munmap_start = mono_ns(); + for (int i = 0; i < a->n; i++) { + if (a->buf[i] == MAP_FAILED) + a->failed++; + else if (munmap(a->buf[i], a->size) != 0) + a->failed++; + } + a->munmap_end = mono_ns(); + a->munmap_ns = (double) (a->munmap_end - a->munmap_start) / a->n; + return NULL; +} + +#define MT_MAX_THREADS 8 +#define MT_REGION_CAP 3000 /* keep T*n well under GUEST_MAX_REGIONS (4096) */ + +static int bench_mt_measure(uint64_t size, + int thread_count, + mt_result_t *result) +{ + int n = MT_REGION_CAP / thread_count; + mt_arg_t arg[MT_MAX_THREADS] = {0}; + pthread_t th[MT_MAX_THREADS]; + int ok = 1, created = 0; + for (int i = 0; i < thread_count; i++) { + arg[i].size = size; + arg[i].n = n; + arg[i].buf = calloc((size_t) n, sizeof(void *)); + if (!arg[i].buf) + ok = 0; + } + + int barrier_ready = 0; + if (ok) { + if (pthread_barrier_init(&mt_barrier, NULL, (unsigned) thread_count) == + 0) + barrier_ready = 1; + else + ok = 0; + } + pthread_mutex_lock(&mt_start_lock); + mt_start = 0; + mt_abort = 0; + pthread_mutex_unlock(&mt_start_lock); + for (int i = 0; i < thread_count && ok; i++) { + if (pthread_create(&th[i], NULL, mt_worker, &arg[i]) != 0) { + ok = 0; + break; + } + created++; + } + pthread_mutex_lock(&mt_start_lock); + mt_abort = !ok; + mt_start = 1; + pthread_cond_broadcast(&mt_start_cond); + pthread_mutex_unlock(&mt_start_lock); + + for (int i = 0; i < created; i++) + pthread_join(th[i], NULL); + + double mmap_sum = 0, munmap_sum = 0; + uint64_t mmap_start = UINT64_MAX, mmap_end = 0; + uint64_t munmap_start = UINT64_MAX, munmap_end = 0; + int failed = 0; + for (int i = 0; i < thread_count; i++) { + mmap_sum += arg[i].mmap_ns; + munmap_sum += arg[i].munmap_ns; + if (arg[i].mmap_start < mmap_start) + mmap_start = arg[i].mmap_start; + if (arg[i].mmap_end > mmap_end) + mmap_end = arg[i].mmap_end; + if (arg[i].munmap_start < munmap_start) + munmap_start = arg[i].munmap_start; + if (arg[i].munmap_end > munmap_end) + munmap_end = arg[i].munmap_end; + failed += arg[i].failed; + free(arg[i].buf); + } + if (barrier_ready) + pthread_barrier_destroy(&mt_barrier); + if (!ok || failed || mmap_end <= mmap_start || munmap_end <= munmap_start) + return 0; + + double total_ops = (double) n * thread_count; + result->mmap_ns = mmap_sum / thread_count; + result->mmap_mops = total_ops * 1000.0 / (double) (mmap_end - mmap_start); + result->munmap_ns = munmap_sum / thread_count; + result->munmap_mops = + total_ops * 1000.0 / (double) (munmap_end - munmap_start); + return 1; +} + +static void bench_mt(void) +{ + static const uint64_t sizes[] = {4 * KIB, 2 * MIB}; + static const int threads[] = {1, 2, 4, 8}; + printf("== Multi-threaded mmap/munmap fast paths ==\n"); + printf("ns is mean per-thread time; Mops/s is aggregate throughput.\n"); + printf("%-10s %8s %8s %12s %12s %12s %12s\n", "size", "threads", "ops/thr", + "mmap ns", "mmap Mops/s", "munmap ns", "munmap Mops/s"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + for (unsigned t = 0; t < sizeof(threads) / sizeof(threads[0]); t++) { + int thread_count = threads[t]; + int n = MT_REGION_CAP / thread_count; + mt_result_t result; + char hb[16]; + if (!bench_mt_measure(sizes[s], thread_count, &result)) { + printf("%-10s %8d %8d %12s\n", human(sizes[s], hb), + thread_count, n, "FAILED"); + continue; + } + printf("%-10s %8d %8d %12.1f %12.3f %12.1f %12.3f\n", + human(sizes[s], hb), thread_count, n, result.mmap_ns, + result.mmap_mops, result.munmap_ns, result.munmap_mops); + } + } + printf("\n"); +} + +/* B. Teardown after materialization. The store is deliberately outside the + * timed interval: it takes the lazy first-touch fault and installs the first + * page, then the counter brackets only munmap(). Each row has exactly one + * materialized 4-KiB page; touching every page would instead benchmark faulting + * and zeroing gigabytes of memory. + * + * bench_munmap_materialized_measure() holds the timing loop for exactly one + * size, shared by the in-process sweep below and the "b-one" isolated-process + * driver mode (see tests/bench-mmap-isolated). + */ +static int bench_munmap_materialized_measure(uint64_t size, + double timer_ticks, + unsigned *reported_ops, + double *munmap_ns) +{ + double unmap_ns[ITERS]; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t unmap_ticks = 0; + unsigned ops = 0; + do { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + p[0] = 1; /* materialize before starting the timed interval */ + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + unmap_ticks += t1 - t0; + if (rc != 0) { + ok = 0; + break; + } + ops++; + } while (unmap_ticks < MIN_TIMED_TICKS); + + if (it >= 0 && ok) { + unmap_ns[it] = + ((double) unmap_ticks / ops - timer_ticks) * ns_per_tick; + *reported_ops = ops; + } + } + if (!ok) + return 0; + *munmap_ns = median(unmap_ns, ITERS); + return 1; +} + +static void bench_munmap_materialized(void) +{ + double timer_ticks = rd_pair_ticks(); + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + + printf("== B. munmap after materializing one 4 KiB page ==\n"); + printf("%-10s %8s %12s\n", "size", "ops", "munmap ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + unsigned reported_ops = 0; + double munmap_ns = 0; + char hb[16]; + if (!bench_munmap_materialized_measure(size, timer_ticks, &reported_ops, + &munmap_ns)) + printf("%-10s %8s %12s\n", human(size, hb), "-", "FAILED"); + else + printf("%-10s %8u %12.1f\n", human(size, hb), reported_ops, + munmap_ns); + } + printf("\n"); +} + +/* C. Teardown after every page was dirtied. Page stores happen before the timed + * interval, so this reports only munmap's handling of the materialized, dirty + * mapping. Periodic VM exits keep large store loops below elfuse's watchdog. + */ +static bool bench_munmap_dirty_once(uint64_t size, + double timer_ticks, + double *unmap_ns) +{ + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return false; + for (uint64_t off = 0; off < size; off += 4 * KIB) { + p[off] = (uint8_t) (off >> 12); + + /* Large already-backed runs can execute in EL0 long enough for the + * benchmark's 10-second vCPU watchdog to fire under macOS memory + * pressure. This deliberately failing fcntl is a guaranteed, + * side-effect-free HVC outside the timed region. It also drains + * retirement in bounded chunks. + */ + if (off != 0 && (off & (DIRTY_VMEXIT_STRIDE - 1)) == 0) + (void) fcntl(-1, F_GETFD); + } + + /* Establish a clean host-retirement boundary before the measurement. */ + (void) fcntl(-1, F_GETFD); + + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + if (rc != 0) + return false; + double ticks = (double) (t1 - t0) - timer_ticks; + if (ticks < 0.0) + ticks = 0.0; + *unmap_ns = ticks * ns_per_tick; + return true; +} + +static int bench_munmap_dirty_measure(uint64_t size, + unsigned ops, + double timer_ticks, + double *p50_ns, + double *p95_ns, + double *max_ns) +{ + double *samples = malloc((size_t) ops * sizeof(*samples)); + if (!samples) + return 0; + + double warmup_ns; + if (!bench_munmap_dirty_once(size, timer_ticks, &warmup_ns)) { + free(samples); + return 0; + } + for (unsigned op = 0; op < ops; op++) { + if (!bench_munmap_dirty_once(size, timer_ticks, &samples[op])) { + free(samples); + return 0; + } + } + + qsort(samples, ops, sizeof(*samples), cmp_d); + *p50_ns = sorted_quantile(samples, ops, 0.50); + *p95_ns = sorted_quantile(samples, ops, 0.95); + *max_ns = samples[ops - 1]; + free(samples); + return 1; +} + +static void bench_munmap_dirty(void) +{ + static const struct { + uint64_t size; + unsigned ops; + } cases[] = { + {4 * KIB, 2048}, {16 * KIB, 2048}, {64 * KIB, 2048}, {256 * KIB, 1024}, + {MIB, 512}, {2 * MIB, 256}, {8 * MIB, 128}, {64 * MIB, 32}, + {256 * MIB, 24}, {GIB, 8}, + }; + double timer_ticks = rd_pair_ticks(); + + printf("== C. munmap after dirtying every 4 KiB page ==\n"); + printf("%-10s %8s %12s %12s %12s\n", "size", "ops", "p50 ns", "p95 ns", + "max ns"); + for (unsigned s = 0; s < sizeof(cases) / sizeof(cases[0]); s++) { + uint64_t size = cases[s].size; + unsigned ops = cases[s].ops; + double p50_ns = 0, p95_ns = 0, max_ns = 0; + char hb[16]; + if (!bench_munmap_dirty_measure(size, ops, timer_ticks, &p50_ns, + &p95_ns, &max_ns)) + printf("%-10s %8u %12s %12s %12s\n", human(size, hb), ops, "FAILED", + "-", "-"); + else + printf("%-10s %8u %12.1f %12.1f %12.1f\n", human(size, hb), ops, + p50_ns, p95_ns, max_ns); + } + printf("\n"); +} + +int main(int argc, char **argv) +{ + clock_init(); + if ((argc == 2 || argc == 3) && strcmp(argv[1], "host-drain") == 0) { + unsigned samples = HOST_DRAIN_DEFAULT_SAMPLES; + if (argc == 3) { + char *end = NULL; + errno = 0; + unsigned long value = strtoul(argv[2], &end, 0); + if (errno || !end || *end || value == 0 || value > 100000) + return 2; + samples = (unsigned) value; + } + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + return bench_host_drain(samples) ? 0 : 1; + } + if (argc == 2 && strcmp(argv[1], "b") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_materialized(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_dirty(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "mt") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_mt(); + return 0; + } + if (argc == 4 && strcmp(argv[1], "mt-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long threads_arg = strtol(argv[3], &end, 0); + if (errno || !end || *end || + (threads_arg != 1 && threads_arg != 2 && threads_arg != 4 && + threads_arg != 8)) + return 2; + int thread_count = (int) threads_arg; + mt_result_t result; + if (!bench_mt_measure(size, thread_count, &result)) { + printf("fastpath-mt size=%llu threads=%d FAILED\n", + (unsigned long long) size, thread_count); + return 1; + } + printf( + "fastpath-mt size=%llu threads=%d ops_per_thread=%d " + "mmap_ns=%.6f mmap_mops=%.6f munmap_ns=%.6f " + "munmap_mops=%.6f\n", + (unsigned long long) size, thread_count, + MT_REGION_CAP / thread_count, result.mmap_ns, result.mmap_mops, + result.munmap_ns, result.munmap_mops); + return 0; + } + if (argc == 2 && strcmp(argv[1], "fault") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault(512, 0); + return 0; + } + if (argc == 3 && strcmp(argv[1], "fault") == 0) { + char *end = NULL; + errno = 0; + long pages = strtol(argv[2], &end, 0); + if (errno || !end || *end || pages < 1 || pages > 1048576) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault((int) pages, 0); + return 0; + } + if (argc == 2 && strcmp(argv[1], "fault-drain") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault(512, 1); + return 0; + } + if (argc == 4 && strcmp(argv[1], "fresh") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long count = strtol(argv[3], &end, 0); + if (errno || !end || *end || count < 1 || count > 1000) + return 2; + return bench_fresh_one(size, (int) count); + } + + /* One section-A sample. The host-side driver starts a new elfuse process + * for every invocation and aggregates the raw one-call timings. + */ + if (argc == 3 && strcmp(argv[1], "a-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + return bench_fastpath_once(size); + } + /* One section-B data point for the host-side driver. */ + if (argc == 3 && strcmp(argv[1], "b-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + double timer_ticks = rd_pair_ticks(); + unsigned reported_ops = 0; + double munmap_ns = 0; + if (!bench_munmap_materialized_measure(size, timer_ticks, &reported_ops, + &munmap_ns)) { + printf("munmap-materialized size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + printf("munmap-materialized size=%llu ops=%u munmap_ns=%.6f\n", + (unsigned long long) size, reported_ops, munmap_ns); + return 0; + } + + /* One warmed section-C sample for the host-side process distribution. */ + if (argc == 3 && strcmp(argv[1], "c-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + double timer_ticks = rd_pair_ticks(); + double warmup_ns = 0; + if (!bench_munmap_dirty_once(size, timer_ticks, &warmup_ns)) { + printf("munmap-dirty size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + /* Drain the warmup retirement before constructing the timed mapping. */ + (void) fcntl(-1, F_GETFD); + + double munmap_ns = 0; + if (!bench_munmap_dirty_once(size, timer_ticks, &munmap_ns)) { + printf("munmap-dirty size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + printf("munmap-dirty size=%llu ops=1 munmap_ns=%.6f\n", + (unsigned long long) size, munmap_ns); + return 0; + } + if (argc != 1) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_materialized(); + bench_munmap_dirty(); + bench_fresh(); + bench_fault(512, 0); + bench_mprotect_split(); + bench_mremap(); + bench_mt(); + return 0; +} diff --git a/tests/lib/bench-mmap.sh b/tests/lib/bench-mmap.sh new file mode 100644 index 00000000..b8cde5b9 --- /dev/null +++ b/tests/lib/bench-mmap.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +human() +{ + awk -v n="$1" 'BEGIN { + if (n >= 1073741824) printf "%g GiB", n / 1073741824 + else if (n >= 1048576) printf "%g MiB", n / 1048576 + else printf "%g KiB", n / 1024 + }' +} + +# field Prints the value of the first "=" token in +# . +field() +{ + local value + if ! value="$(awk -v key="$2" '{ + for (i = 1; i <= NF; i++) { + if (index($i, key "=") == 1) { + sub("^" key "=", "", $i) + print $i + found = 1 + exit + } + } + } + END { + if (!found) + exit 1 + }' <<< "$1")"; then + printf 'benchmark output missing %s field:\n%s\n' "$2" "$1" >&2 + return 1 + fi + printf '%s\n' "$value" +} + +numeric_median() +{ + printf '%s\n' "$@" | LC_ALL=C sort -n | awk '{ values[NR] = $1 } + END { + if (NR % 2) + print values[(NR + 1) / 2] + else + print (values[NR / 2] + values[NR / 2 + 1]) / 2 + }' +} diff --git a/tests/manifest.txt b/tests/manifest.txt index 8ad6cce6..dd06ac64 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -134,6 +134,10 @@ test-guard-page test-mmap-hint test-mmap-sigbus-efault +[section] Lazy anonymous mmap tests +test-mmap-lazy +test-mmap-fastpath + [section] mremap tests test-mremap test-mremap-infra diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 30d64bde..f36be99f 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -22,9 +22,10 @@ #define PREVIOUS_ELFN_MAGIC 0x454C464EU #define PREVIOUS_ELFO_MAGIC 0x454C464FU #define PREVIOUS_ELFP_MAGIC 0x454C4650U +#define PREVIOUS_ELFQ_MAGIC 0x454C4651U -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4651U, - "fork IPC protocol magic must remain ELFQ until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4652U, + "fork IPC protocol magic must remain ELFR until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -40,6 +41,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFO_MAGIC, "per-fd description ownership requires rejecting ELFO peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFP_MAGIC, "the per-fd intercept marker requires rejecting ELFP peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFQ_MAGIC, + "the dirty bitmap wire requires rejecting ELFQ peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-fuse-basic.c b/tests/test-fuse-basic.c index 0aa0679e..d324e4a8 100644 --- a/tests/test-fuse-basic.c +++ b/tests/test-fuse-basic.c @@ -264,12 +264,29 @@ static size_t append_dirent(uint8_t *buf, static void *daemon_main(void *arg) { daemon_ctx_t *ctx = arg; - uint8_t buf[4096]; + const size_t map_len = 4UL << 20; +#if defined(__x86_64__) + /* Rosetta serializes mmap; allocate before serving a file-mmap request. */ + void *mapping = mmap(NULL, map_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + exit(1); +#endif for (;;) { - ssize_t nr = read(ctx->fusefd, buf, sizeof(buf)); +#if !defined(__x86_64__) + /* Each request must reach a buffer the daemon has never touched. */ + void *mapping = mmap(NULL, map_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + exit(1); +#endif + uint8_t *buf = (uint8_t *) mapping + (2UL << 20); + ssize_t nr = read(ctx->fusefd, buf, 4096); if (nr < 0) { - if (errno == ENOTCONN || errno == EBADF) + if (errno == ENOTCONN || errno == EBADF) { + munmap(mapping, map_len); return NULL; + } perror("read(/dev/fuse)"); exit(1); } @@ -281,6 +298,7 @@ static void *daemon_main(void *arg) if (reply_frame(ctx->fusefd, in->unique, -ctx->init_error, NULL, 0) < 0) exit(1); + munmap(mapping, map_len); return NULL; } struct fuse_init_out out = { @@ -439,6 +457,9 @@ static void *daemon_main(void *arg) exit(1); break; } +#if !defined(__x86_64__) + munmap(mapping, map_len); +#endif } } @@ -740,13 +761,35 @@ int main(void) die("lseek(fuse-file)"); expect_hello_fd(fd); - void *map = mmap(NULL, 4096, PROT_READ, MAP_PRIVATE, fd, 0); + void *map; +#if !defined(__x86_64__) + map = mmap(NULL, 4096, PROT_READ, MAP_PRIVATE, fd, 0); if (map != MAP_FAILED || errno != ENODEV) { fprintf(stderr, "expected mmap ENODEV on FUSE fd, got map=%p errno=%d\n", map, errno); return 1; } +#else + /* Rosetta materialization waits for this daemon without holding mmap_lock. + */ + void *fixed = (void *) (uintptr_t) (1ULL << 40); + map = + mmap(fixed, 4096, PROT_READ, MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + if (map != fixed || memcmp(map, hello_data, sizeof(hello_data) - 1) != 0) { + fprintf(stderr, "FUSE high-VA materialization failed: %p errno=%d\n", + map, errno); + return 1; + } + void *conflict = + mmap(fixed, 4096, PROT_READ, MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + if (conflict != MAP_FAILED || errno != EEXIST || + memcmp(map, hello_data, sizeof(hello_data) - 1) != 0) { + fprintf(stderr, "FUSE high-VA NOREPLACE changed an existing mapping\n"); + return 1; + } + munmap(map, 4096); +#endif close(fd); /* Canonicalization: ./ and intermediate-up traversals must collapse to the diff --git a/tests/test-materialize-host.c b/tests/test-materialize-host.c new file mode 100644 index 00000000..bd2d77e5 --- /dev/null +++ b/tests/test-materialize-host.c @@ -0,0 +1,306 @@ +/* + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include "core/guest.h" +#include "runtime/thread.h" +#include "syscall/internal.h" +#include "syscall/mem.h" +#include "syscall/signal.h" +#include "test-harness.h" +#include "utils.h" + +int passes, fails; + +typedef struct { + pthread_cond_t cond; + bool ready; +} lock_wait_t; + +static void *wake_lock_waiter(void *arg) +{ + lock_wait_t *wait = arg; + mmap_lock_acquire_raw(); + wait->ready = true; + pthread_cond_signal(&wait->cond); + mmap_lock_release_raw(); + return NULL; +} + +static void test_nofault(guest_t *g) +{ + const uint64_t block = 2ULL << 20; + const uint64_t addr = 32ULL << 20; + if (guest_region_add( + g, addr, addr + 2 * block, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL) < 0) { + TEST("nofault setup"); + FAIL("could not add lazy region"); + return; + } + uint8_t data[2] = {0xa5, 0x5a}; + TEST("raw mmap lock tracks ownership"); + mmap_lock_acquire_raw(); + bool refused = mmap_lock_held_by_current_thread() && + guest_read(g, addr, data, sizeof(data)) < 0; + mmap_lock_release_raw(); + EXPECT_TRUE(refused && !mmap_lock_held_by_current_thread(), + "raw scope allowed recursive materialization"); + + TEST("condition wait restores mmap ownership"); + lock_wait_t wait = {.cond = PTHREAD_COND_INITIALIZER}; + pthread_t worker; + mmap_lock_acquire(g); + int started = pthread_create(&worker, NULL, wake_lock_waiter, &wait); + while (started == 0 && !wait.ready) + mmap_lock_cond_wait(g, &wait.cond); + refused = mmap_lock_held_by_current_thread() && + guest_read(g, addr, data, sizeof(data)) < 0; + mmap_lock_release(); + if (started == 0) + pthread_join(worker, NULL); + pthread_cond_destroy(&wait.cond); + EXPECT_TRUE(started == 0 && refused, "condwait lost ownership tracking"); + + TEST("nofault does not materialize"); + EXPECT_TRUE(guest_read_nofault(g, addr, data, sizeof(data)) < 0 && + guest_write_nofault(g, addr, data, sizeof(data)) < 0 && + !guest_va_pte_valid(g, addr), + "nofault published a lazy PTE"); + + TEST("partial nofault stops at lazy block"); + int rc = guest_lazy_faultin(g, addr, 1); + size_t copied = guest_write_partial_nofault(g, addr + block - 1, data, 2); + EXPECT_TRUE(rc == 0 && copied == 1 && !guest_va_pte_valid(g, addr + block), + "partial copy lost its boundary or faulted in the next block"); + + TEST("prefault then nofault crosses blocks"); + rc = guest_lazy_faultin(g, addr + block - 1, 2); + uint8_t readback[2] = {0}; + mmap_lock_acquire(g); + bool copied_all = guest_write_nofault(g, addr + block - 1, data, 2) == 0 && + guest_read_nofault(g, addr + block - 1, readback, 2) == 0; + mmap_lock_release(); + EXPECT_TRUE(rc == 0 && copied_all && readback[0] == data[0] && + readback[1] == data[1], + "prepared cross-block copy failed"); + + TEST("locked missing range returns failure"); + mmap_lock_acquire(g); + rc = guest_lazy_faultin_locked(g, addr + 4 * block, 1); + refused = guest_read(g, addr + 4 * block, data, 1) < 0 && + guest_read_nofault(g, addr + 4 * block, data, 1) < 0; + mmap_lock_release(); + EXPECT_TRUE(rc < 0 && refused, "missing range did not fail safely"); +} + +static void test_prepared_mmap(guest_t *g) +{ + TEST("prepared mmap does not look up guest fd"); + char path[] = "/tmp/elfuse-prepared-mmap-XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + const uint8_t marker = 0xa5; + bool ok = ftruncate(fd, GUEST_PAGE_SIZE) == 0 && write(fd, &marker, 1) == 1; + const uint64_t addr = 1ULL << 40; + g->is_rosetta = true; + mmap_lock_acquire(g); + int64_t mapped = + ok ? sys_mmap(g, addr, GUEST_PAGE_SIZE, LINUX_PROT_READ, + LINUX_MAP_PRIVATE | LINUX_MAP_FIXED, -1, 0, fd) + : -1; + mmap_lock_release(); + uint8_t value = 0; + ok = mapped == (int64_t) addr && guest_read(g, addr, &value, 1) == 0 && + value == marker; + if (mapped == (int64_t) addr) { + mmap_lock_acquire(g); + int64_t conflict = + sys_mmap(g, addr, GUEST_PAGE_SIZE, LINUX_PROT_READ, + LINUX_MAP_PRIVATE | LINUX_MAP_FIXED_NOREPLACE, -1, 0, fd); + ok = ok && conflict == -LINUX_EEXIST; + close(fd); + fd = -1; + const guest_region_t *region = guest_region_find(g, addr); + ok = ok && region && pread(region->backing_fd, &value, 1, 0) == 1 && + value == marker; + ok = sys_munmap(g, addr, GUEST_PAGE_SIZE) == 0 && ok; + mmap_lock_release(); + } + if (fd >= 0) + close(fd); + g->is_rosetta = false; + EXPECT_TRUE(ok, "prepared fd or commit-time overlap check failed"); +} + +static void test_mremap_lazy_source(guest_t *g) +{ + const uint64_t block = 2ULL << 20; + const uint64_t base = 64ULL << 20; + const uint64_t source = base + 4 * GUEST_PAGE_SIZE; + const uint64_t dest = base + 16 * GUEST_PAGE_SIZE; + const uint64_t len = 3 * GUEST_PAGE_SIZE; + const int flags = + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE; + uint8_t *slab = g->host_base; + TEST("mremap preserves valid L3 pages among dirty lazy holes"); + mmap_lock_acquire(g); + bool ok = + guest_region_add(g, source, source + len, + LINUX_PROT_READ | LINUX_PROT_WRITE, flags, 0, + NULL) == 0 && + guest_extend_page_tables(g, base, base + block, MEM_PERM_RW) == 0 && + guest_split_block(g, base) == 0 && + guest_invalidate_ptes(g, base, base + block) == 0 && + guest_update_perms(g, source + GUEST_PAGE_SIZE, + source + 2 * GUEST_PAGE_SIZE, MEM_PERM_RW) == 0; + memset(slab + base, 0xa5, block); + guest_dirty_mark_range(g, base, base + block); + int64_t result = + ok ? sys_mremap(g, source, len, len, + LINUX_MREMAP_MAYMOVE | LINUX_MREMAP_FIXED, dest) + : -1; + ok = ok && result == (int64_t) dest && !guest_va_pte_valid(g, source) && + !guest_va_pte_valid(g, dest - GUEST_PAGE_SIZE) && + !guest_va_pte_valid(g, dest + len); + for (uint64_t off = 0; ok && off < len; off++) { + uint8_t expected = + off >= GUEST_PAGE_SIZE && off < 2 * GUEST_PAGE_SIZE ? 0xa5 : 0; + ok = slab[dest + off] == expected; + } + mmap_lock_release(); + EXPECT_TRUE(ok, + "copy exposed lazy bytes, lost valid data, or mapped a guard"); + + TEST("low-address mremap growth zeroes only the source edge block"); + const uint64_t low = 96ULL << 20; + const uint64_t old_len = 2 * block + GUEST_PAGE_SIZE; + mmap_lock_acquire(g); + ok = guest_region_add(g, low, low + old_len, + LINUX_PROT_READ | LINUX_PROT_WRITE, flags, 0, + NULL) == 0 && + guest_invalidate_ptes(g, low, low + old_len + GUEST_PAGE_SIZE) == 0; + memset(slab + low, 0xa5, old_len + GUEST_PAGE_SIZE); + guest_dirty_mark_range(g, low, low + old_len + GUEST_PAGE_SIZE); + result = + ok ? sys_mremap(g, low, old_len, old_len + GUEST_PAGE_SIZE, 0, 0) : -1; + ok = ok && result == (int64_t) low && !guest_va_pte_valid(g, low) && + slab[low] == 0xa5 && + guest_va_pte_valid(g, low + old_len - GUEST_PAGE_SIZE); + for (uint64_t off = old_len - GUEST_PAGE_SIZE; + ok && off < old_len + GUEST_PAGE_SIZE; off++) + ok = slab[low + off] == 0; + mmap_lock_release(); + EXPECT_TRUE(ok, "growth exposed dirty bytes or materialized the old body"); +} + +int main(void) +{ + guest_t *g = calloc(1, sizeof(*g)); + if (!g || guest_init(g, 64ULL << 30, 0) != 0) + return 1; + if (!guest_build_page_tables(g, NULL, 0)) + return 1; + const uint64_t block = 2ULL << 20; + const uint64_t addr = 16ULL << 20; + if (guest_region_add( + g, addr, addr + block, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL) < 0) + return 1; + + TEST("zero-length resolve preserves lazy PTEs"); + uint64_t avail = 0; + void *ptr = guest_ptr_bound(g, addr, &avail, MEM_PERM_R, 0); + EXPECT_TRUE(ptr == NULL && !guest_va_pte_valid(g, addr) && + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] == 0, + "empty resolve materialized memory"); + + TEST("locked guest access refuses lazy faults"); + uint8_t value = 0xa5; + mmap_lock_acquire(g); + bool refused = guest_read(g, addr, &value, sizeof(value)) < 0 && + guest_write(g, addr, &value, sizeof(value)) < 0 && + guest_ptr(g, addr) == NULL && !guest_va_pte_valid(g, addr); + mmap_lock_release(); + EXPECT_TRUE(refused, "locked resolve entered lazy materialization"); + + TEST("writable materialization remains dirty"); + uint8_t *word = guest_ptr_bound(g, addr, &avail, MEM_PERM_W, 1); + if (!word) + return 1; + *word = 0xa5; + mmap_lock_acquire(g); + bool dirty = guest_block_may_be_dirty(g, addr); + int rc = guest_invalidate_ptes(g, addr, addr + block); + if (rc == 0) + rc = guest_materialize_lazy(g, addr); + bool zero = *word == 0; + dirty = dirty && guest_block_may_be_dirty(g, addr); + mmap_lock_release(); + EXPECT_TRUE(rc == 0 && zero && dirty, + "writable reuse skipped zeroing or lost dirty state"); + + TEST("read-only zeroed block can clean-skip on reuse"); + mmap_lock_acquire(g); + g->regions[0].prot = LINUX_PROT_READ; + rc = guest_invalidate_ptes(g, addr, addr + block); + if (rc == 0) + rc = guest_materialize_lazy(g, addr); + uint64_t skips = g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP]; + if (rc == 0) + rc = guest_invalidate_ptes(g, addr, addr + block); + if (rc == 0) + rc = guest_materialize_lazy(g, addr); + bool clean = !guest_block_may_be_dirty(g, addr); + bool skipped = g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP] > skips; + mmap_lock_release(); + EXPECT_TRUE(rc == 0 && clean && skipped, + "clean reuse did not skip zeroing"); + + test_nofault(g); + test_prepared_mmap(g); + test_mremap_lazy_source(g); + + TEST("signal frame fits below low SP without prefault slack"); + if (guest_region_add( + g, 0, block, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL) < 0) + return 1; + hv_vcpu_exit_t *vexit; + if (hv_vcpu_create(&g->vcpu, &vexit, NULL) != HV_SUCCESS) + return 1; + g->vcpu_valid = true; + thread_init(); + signal_init(); + signal_state_snapshot_t *state = calloc(1, sizeof(*state)); + if (!state) + return 1; + state->actions[LINUX_SIGUSR1 - 1].sa_handler = addr; + signal_set_state(state); + free(state); + hv_vcpu_set_sys_reg(g->vcpu, HV_SYS_REG_SP_EL0, + sizeof(linux_rt_sigframe_t)); + signal_queue(LINUX_SIGUSR1); + int exit_code = 0; + rc = signal_deliver(g->vcpu, g, &exit_code); + uint64_t sp = UINT64_MAX; + hv_vcpu_get_sys_reg(g->vcpu, HV_SYS_REG_SP_EL0, &sp); + EXPECT_TRUE(rc == 1 && sp == 0 && guest_va_pte_valid(g, 0), + "low-SP signal frame was not installed"); + + guest_destroy(g); + free(g); + SUMMARY("test-materialize-host"); + return fails != 0; +} diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 8f850b53..bfe403ca 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -644,24 +644,23 @@ test_pipe() # # This is the full aarch64 unit-test surface: every tests/manifest.txt ("make # check") binary except the handful that assert elfuse-internal implementation -# details with no meaningful counterpart on a real kernel (most of the EL1 shim -# fast-path suite -- test-shim-* and test-shim-cred-race, which probe elfuse's -# own shim_data block and identity cache; test-shim-futex-fast is the exception -# and does run here, because every assertion in it is plain Linux futex ABI that -# a real kernel adjudicates (unlike test-mremap-infra, which guards elfuse's -# guest-IPA infra reserve, and test-oom-proc, documented in its own header). -# test-mremap-tail-emfile is listed here as an elfuse-lane regression and marked -# QEMU_SKIP because its host-reserve assertion has no Linux analogue. There is -# no "core" vs "extended" split here; everything below runs in both -# elfuse-aarch64 and qemu-aarch64 modes, and genuine, understood divergences -# from the qemu reference kernel are called out via QEMU_SKIP with a comment -# rather than silently dropped from this list. The unit lane runs binaries this -# repo builds, not fixtures it downloads, so an empty build/ is a setup mistake -# rather than a run to report on. Without this every test fails on a missing -# file and the summary reads like a hundred-odd regressions; "make clean" -# followed by "make elfuse" is enough to produce it, because elfuse alone does -# not build the test binaries. Name the cause once and stop, the way driver.sh -# does with ALLOW_MISSING_BINARIES. +# details with no meaningful counterpart on a real kernel. Most test-shim-* +# binaries probe elfuse's own shim_data block and identity cache; +# test-shim-futex-fast is included because it asserts Linux futex semantics. +# test-mmap-fastpath probes elfuse's arena and TLBI protocol; test-mremap-infra +# guards elfuse's guest-IPA infra reserve; test-oom-proc documents its reason in +# its own header. test-mremap-tail-emfile is listed here as an elfuse-lane +# regression and marked QEMU_SKIP because its host-reserve assertion has no +# Linux analogue. There is no "core" vs "extended" split here; everything below +# runs in both elfuse-aarch64 and qemu-aarch64 modes, and genuine, understood +# divergences from the qemu reference kernel are called out via QEMU_SKIP with a +# comment rather than silently dropped from this list. The unit lane runs +# binaries this repo builds, not fixtures it downloads, so an empty build/ is a +# setup mistake rather than a run to report on. Without this every test fails on +# a missing file and the summary reads like a hundred-odd regressions; "make +# clean" followed by "make elfuse" is enough to produce it, because elfuse alone +# does not build the test binaries. Name the cause once and stop, the way +# driver.sh does with ALLOW_MISSING_BINARIES. require_unit_binaries() { local bindir="$1" @@ -863,6 +862,7 @@ run_unit_tests() test_rc "$runner" "test-mmap-hint" 0 "$bindir/test-mmap-hint" test_rc "$runner" "test-mmap-sigbus-efault" 0 "$bindir/test-mmap-sigbus-efault" + test_rc "$runner" "test-mmap-lazy" 0 "$bindir/test-mmap-lazy" printf "\nLow-base ET_EXEC memory regression\n" test_rc "$runner" "test-lowbase-mem-200000" 0 "$bindir/test-lowbase-mem-200000" diff --git a/tests/test-mmap-dirty-stats.sh b/tests/test-mmap-dirty-stats.sh new file mode 100755 index 00000000..6ec5cf67 --- /dev/null +++ b/tests/test-mmap-dirty-stats.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Counter-backed dirty-map materialization integration checks. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-lazy} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-dirty-map.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" \ + > "$TMPDIR_CASE/out" 2> "$TMPDIR_CASE/err" + +counter() +{ + key=$1 + awk -v key="$key" \ + '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/err" +} + +require_ge() +{ + key=$1 + floor=$2 + value=$(counter "$key") || { + printf 'missing dirty-map counter %s\n' "$key" >&2 + return 1 + } + if [ "$value" -lt "$floor" ]; then + printf '%s=%s, expected >= %s\n' "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_ge FAULT_CLEAN_SKIP 1 +require_ge FAULT_DIRTY_MEMSET 1 +require_ge FAULT_ALREADY_VALID 1 +# One full BLOCK_2MIB materialization accounts for 2 MiB of window bytes. +require_ge FAULT_WINDOW_BYTES 2097152 + +printf ' clean-block zero skip OK\n' +printf ' dirty-block selective memset OK\n' +printf ' already-valid early return OK\n' +printf ' materialized-window bytes OK\n' +printf 'test-mmap-dirty-stats: PASS\n' diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh new file mode 100755 index 00000000..14581194 --- /dev/null +++ b/tests/test-mmap-fastpath-stats.sh @@ -0,0 +1,172 @@ +#!/bin/sh +# Counter-backed refill, adaptive sizing, giant-request guard, and VA recycle +# integration checks for the EL1 anonymous-mmap consumer fast path. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-fastpath} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-mmap-stats.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +run_case() +{ + case_name=$1 + out="$TMPDIR_CASE/$case_name.out" + err="$TMPDIR_CASE/$case_name.err" + ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" "--stats-$case_name" \ + > "$out" 2> "$err" +} + +counter() +{ + case_name=$1 + key=$2 + value=$(awk -v key="$key" '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/$case_name.err") || { + printf 'missing counter %s in case %s\n' "$key" "$case_name" >&2 + return 1 + } + printf '%s\n' "$value" +} + +require_ge() +{ + case_name=$1 + key=$2 + floor=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -lt "$floor" ]; then + printf '%s: %s=%s, expected >= %s\n' \ + "$case_name" "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_eq() +{ + case_name=$1 + key=$2 + expected=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -ne "$expected" ]; then + printf '%s: %s=%s, expected %s\n' \ + "$case_name" "$key" "$value" "$expected" >&2 + return 1 + fi +} + +require_le() +{ + case_name=$1 + key=$2 + ceiling=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -gt "$ceiling" ]; then + printf '%s: %s=%s, expected <= %s\n' \ + "$case_name" "$key" "$value" "$ceiling" >&2 + return 1 + fi +} + +run_case ring-full +require_ge ring-full MMAP_HIT 32 +require_ge ring-full MMAP_RING_FULL 1 +printf ' 32-entry ring fallback OK\n' + +run_case np2-10m +require_ge np2-10m MMAP_HIT 80 +require_ge np2-10m MMAP_CAPACITY_MISS 1 +printf ' sustained 10 MiB stream OK\n' + +run_case np2-48m +require_ge np2-48m MMAP_HIT 40 +require_ge np2-48m MMAP_CAPACITY_MISS 1 +printf ' sustained 48 MiB stream OK\n' + +run_case np2-100m +require_ge np2-100m MMAP_HIT 24 +require_ge np2-100m MMAP_CAPACITY_MISS 1 +printf ' sustained 100 MiB stream OK\n' + +run_case escalation +require_ge escalation MMAP_HIT 45 +require_eq escalation MMAP_ARENA_CURRENT 17179869184 +printf ' 10 MiB -> 512 MiB escalation OK\n' + +run_case giant-guard +require_ge giant-guard MMAP_HIT 34 +require_eq giant-guard MMAP_ARENA_PEAK 34359738368 +printf ' 2 GiB request uses fast path OK\n' + +run_case adaptive-small +require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-small MMAP_ARENA_PEAK 67108864 +printf ' small-stream arena floor OK\n' + +run_case adaptive-retention +require_eq adaptive-retention MMAP_ARENA_CURRENT 17179869184 +require_eq adaptive-retention MMAP_ARENA_PEAK 17179869184 +printf ' large arena retained OK\n' + +run_case adaptive-rewind-growth +require_ge adaptive-rewind-growth MMAP_CAPACITY_MISS 1 +require_eq adaptive-rewind-growth MMAP_ARENA_CURRENT 268435456 +printf ' rewound arena grows to target OK\n' + +run_case recycle +require_ge recycle MMAP_RECYCLE 1 +require_le recycle MMAP_HIGH_WATER 201326592 +printf ' arena VA recycling OK\n' + +# Mixed-size churn now recycles VA through the arena rewind and the host gap +# allocator alone. MMAP_RECYCLE is the load-bearing assertion: it is nonzero +# only when a refill actually reclaimed a hole below the high-water mark, so it +# fails the moment VA recovery stops and the allocator only walks forward. The +# hit count keeps the case honest because the other two would also pass if the +# fast path stopped being taken at all. The bound catches growth that recovery +# is too slow to contain. +run_case mixed-churn +require_ge mixed-churn MMAP_HIT 100 +require_ge mixed-churn MMAP_RECYCLE 1 +require_le mixed-churn MMAP_HIGH_WATER 402653184 +printf ' mixed-size churn recycles VA OK\n' + +# A first touch enters the host while the 32 GiB mapping is still live. Its +# arena descriptor must survive that VM exit so the following munmap remains an +# EL1 retirement instead of stranding the mapping in a speculative top-up. +run_case large-materialized +require_eq large-materialized MMAP_ARENA_PEAK 34359738368 +require_le large-materialized MMAP_HIGH_WATER 38654705664 +printf ' large live arena is retained OK\n' + +# Fork drains and revokes every arena. A near-empty current arena must not be +# topped up between those operations; only the initial vCPU arena is counted. +run_case fork-no-topup +require_eq fork-no-topup MMAP_HIT 1 +require_eq fork-no-topup MMAP_REFILL 1 +printf ' fork skips arena top-up OK\n' + +run_case prefix-hint +require_eq prefix-hint MMAP_REFILL 1 +printf ' prefix hint preserves arena OK\n' + +run_case unused-altstack +require_eq unused-altstack FAULT_WINDOW_BYTES 0 +run_case used-altstack +require_ge used-altstack FAULT_WINDOW_BYTES 4096 +printf ' only selected altstack faults in OK\n' + +run_case invalid-futex +require_eq invalid-futex FAULT_WINDOW_BYTES 0 +printf ' invalid futex stays lazy OK\n' + +# The test synchronizes through atomics while sibling vCPUs retain control, so +# msync, W^X, and maps/smaps are the first host observers of fast publications +# and retirements. Keep the case counter-backed so a disabled fast path cannot +# turn the semantic checks into a false green. +run_case observer-semantics +require_ge observer-semantics MMAP_HIT 1 +printf ' observer semantics under churn OK\n' + +printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c new file mode 100644 index 00000000..3254af62 --- /dev/null +++ b/tests/test-mmap-fastpath.c @@ -0,0 +1,1198 @@ +/* + * EL1 consumer-mmap fast-path integration tests. + * + * Run through the dedicated make target without an ELFUSE_MMAP_FASTPATH + * override so the default-enabled configuration is exercised. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" +#include "raw-syscall.h" + +int passes = 0, fails = 0; + +static sigjmp_buf segv_jmp; + +static inline void spin_hint(void); + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static int proc_extent_for(const char *path, + uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + size_t cap = 16384, used = 0; + char *buf = malloc(cap); + if (!buf) { + close(fd); + return -1; + } + for (;;) { + if (used == cap - 1) { + if (cap >= 4 * 1024 * 1024) { + free(buf); + close(fd); + return -1; + } + size_t next_cap = cap * 2; + char *next = realloc(buf, next_cap); + if (!next) { + free(buf); + close(fd); + return -1; + } + buf = next; + cap = next_cap; + } + ssize_t n = read(fd, buf + used, cap - used - 1); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + free(buf); + close(fd); + return -1; + } + if (n == 0) + break; + used += (size_t) n; + } + close(fd); + if (used == 0) { + free(buf); + return -1; + } + buf[used] = '\0'; + + char *line = buf; + while (*line) { + unsigned long long lo, hi; + if (sscanf(line, "%llx-%llx", &lo, &hi) == 2 && needle >= lo && + needle < hi) { + *lo_out = (uintptr_t) lo; + *hi_out = (uintptr_t) hi; + free(buf); + return 1; + } + char *nl = strchr(line, '\n'); + if (!nl) + break; + line = nl + 1; + } + free(buf); + return 0; +} + +static int maps_extent_for(uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + return proc_extent_for("/proc/self/maps", needle, lo_out, hi_out) == 1 ? 0 + : -1; +} + +static void test_fidelity(void) +{ + TEST("unconsumed arena is absent and faults"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + p[0] = 0x5a; /* drains the publication through the fault-side lock */ + + volatile uint8_t *unconsumed = p + 4096; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *unconsumed; + FAIL("wild read into unconsumed arena did not SIGSEGV"); + munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + uintptr_t lo = 0, hi = 0; + if (maps_extent_for((uintptr_t) p, &lo, &hi) < 0 || lo != (uintptr_t) p || + hi != (uintptr_t) p + 4096) { + FAIL("/proc/self/maps exposed more than the consumed page"); + munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + if (munmap(p, 4096) != 0) { + FAIL("munmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene between munmap and this load: EL1 must have + * invalidated the Stage-1 descriptor and completed broadcast TLBI before + * returning, even though host region cleanup is still deferred. + */ + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *(volatile uint8_t *) p; + FAIL("access immediately after munmap did not SIGSEGV"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + sigaction(SIGSEGV, &old_sa, NULL); + PASS(); +} + +static void test_file_mmap_avoids_arena_hole(int flags) +{ + TEST(flags == MAP_SHARED ? "shared mmap avoids live arena hole" + : "private mmap avoids live arena hole"); + const size_t chunk = 2ULL << 20; + char path[] = "/tmp/elfuse-arena-hole-XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + if (ftruncate(fd, (off_t) chunk) != 0) { + close(fd); + FAIL("ftruncate"); + return; + } + + uint8_t *arena = mmap(NULL, 4 * chunk, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (arena == MAP_FAILED) { + close(fd); + FAIL("anonymous mmap"); + return; + } + arena[0] = 0x31; + arena[3 * chunk] = 0x42; + if (munmap(arena + chunk, chunk) != 0) { + munmap(arena, 4 * chunk); + close(fd); + FAIL("munmap arena hole"); + return; + } + + /* The live neighbors keep the arena from rewinding. A NULL hint leaves its + * descriptor enabled while the host searches the retired hole. + */ + uint8_t *file = mmap(NULL, chunk, PROT_READ | PROT_WRITE, flags, fd, 0); + if (file == MAP_FAILED) { + munmap(arena, 4 * chunk); + close(fd); + FAIL("file mmap"); + return; + } + int overlaps = (uintptr_t) file < (uintptr_t) arena + 4 * chunk && + (uintptr_t) file + chunk > (uintptr_t) arena; + file[0] = 0x53; + int intact = arena[0] == 0x31 && arena[3 * chunk] == 0x42; + int rc = munmap(file, chunk); + close(fd); /* drain any EL1 retirement of the file mapping */ + int cleanup_rc = munmap(arena, 4 * chunk); + if (overlaps) + FAIL("file mapping entered the live arena"); + else if (!intact) + FAIL("live anonymous mappings changed"); + else if (rc != 0 || cleanup_rc != 0) + FAIL("munmap cleanup"); + else + PASS(); +} + +static void test_exhaustion_fallback(void) +{ + TEST("arena exhaustion falls back to host mmap"); + const size_t len = 80ULL << 20; /* larger than the first 64MiB arena */ + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("80MiB mmap"); + return; + } + if (p[0] != 0 || p[len - 1] != 0) { + FAIL("fallback mapping was not zero-filled"); + munmap(p, len); + return; + } + p[0] = 1; + p[len - 1] = 2; + if (munmap(p, len) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +typedef struct { + volatile uint8_t *p; + _Atomic int ready; + _Atomic int go; + _Atomic int result; +} sibling_tlbi_arg_t; + +static void *sibling_tlbi_worker(void *opaque) +{ + sibling_tlbi_arg_t *arg = opaque; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) arg->p[0]; /* seed a translation on this sibling vCPU */ + atomic_store_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + (void) arg->p[0]; + atomic_store_explicit(&arg->result, -1, memory_order_release); + } else { + atomic_store_explicit(&arg->result, 1, memory_order_release); + } + return NULL; +} + +static const char *run_sibling_tlbi(size_t len, size_t materialize_stride) +{ + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) + return "sigaction"; + + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + sigaction(SIGSEGV, &old_sa, NULL); + return "mmap"; + } + + for (size_t off = 0; off < len; off += materialize_stride) + p[off] = (uint8_t) (off >> 21); + + sibling_tlbi_arg_t arg = {.p = p}; + pthread_t worker; + if (pthread_create(&worker, NULL, sibling_tlbi_worker, &arg) != 0) { + munmap((void *) p, len); + sigaction(SIGSEGV, &old_sa, NULL); + return "pthread_create"; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + if (munmap((void *) p, len) != 0) { + atomic_store_explicit(&arg.go, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return "munmap"; + } + + /* No syscall may intervene here: the sibling must observe EL1's + * inner-shareable invalidation before host metadata removal. + */ + atomic_store_explicit(&arg.go, 1, memory_order_release); + int result; + while (!(result = atomic_load_explicit(&arg.result, memory_order_acquire))) + spin_hint(); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return result < 0 ? "stale sibling translation survived TLBI" : NULL; +} + +static void test_l3_range_tlbi(void) +{ + TEST("RVALE1IS invalidates sibling L3 translation"); + const char *error = run_sibling_tlbi(16ULL << 10, 4096); + if (error) { + FAIL(error); + return; + } + PASS(); +} + +static void test_large_l2_range_tlbi(void) +{ + TEST("SCALE=3 RVALE1IS invalidates sibling L2 translation"); + const size_t len = 320ULL << 20; /* exceeds SCALE=2's 256MiB maximum */ + const char *error = run_sibling_tlbi(len, 2ULL << 20); + if (error) { + FAIL(error); + return; + } + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int ready; + _Atomic int done; + _Atomic int release; +} handoff_arg_t; + +static inline void spin_hint(void) +{ + __asm__ volatile("yield" ::: "memory"); +} + +static void *handoff_worker(void *opaque) +{ + handoff_arg_t *arg = opaque; + uint8_t *warm = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warm == MAP_FAILED) { + atomic_store_explicit(&arg->done, -1, memory_order_release); + atomic_store_explicit(&arg->ready, 1, memory_order_release); + return NULL; + } + warm[0] = 1; + atomic_store_explicit(&arg->ready, 1, memory_order_release); + uintptr_t ptr; + while (!(ptr = atomic_load_explicit(&arg->ptr, memory_order_acquire))) + spin_hint(); + int rc = munmap((void *) ptr, 4096); + atomic_store_explicit(&arg->done, rc == 0 ? 1 : -1, memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + munmap(warm, 4096); + return NULL; +} + +static void test_cross_vcpu_handoff(void) +{ + TEST("cross-vCPU mmap publication then munmap retirement"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + handoff_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, handoff_worker, &arg) != 0) { + FAIL("pthread_create"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + /* Keep the worker alive after munmap so its next thread-exit syscall cannot + * drain either ring. The fault below is the first natural VM exit after A's + * mmap publication and B's retirement. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + atomic_store_explicit(&arg.ptr, (uintptr_t) p, memory_order_release); + int done; + while (!(done = atomic_load_explicit(&arg.done, memory_order_acquire))) + spin_hint(); + + int faulted = 0; + if (done > 0 && sigsetjmp(segv_jmp, 1) == 0) + (void) *(volatile uint8_t *) p; + else if (done > 0) + faulted = 1; + + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (done < 0) { + FAIL("worker munmap"); + return; + } + if (!faulted) { + FAIL("retired cross-vCPU mapping remained accessible"); + return; + } + PASS(); +} + +static void test_mixed_size_recycled_va_reads_zero(void) +{ + TEST("recycled mixed-size VA reads zero"); + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + + for (int i = 0; i < 128; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + if (p[0] != 0 || p[len / 2] != 0 || p[len - 1] != 0) { + munmap((void *) p, len); + FAIL("recycled VA exposed stale bytes"); + return; + } + p[0] = (uint8_t) (i + 1); + p[len / 2] = (uint8_t) (i ^ 0x5a); + p[len - 1] = (uint8_t) (i ^ 0xa5); + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + return; + } + } + PASS(); +} + +static void test_repeated_munmap_does_not_alias(void) +{ + TEST("repeated munmap does not alias two live mappings"); + const size_t len = 64ULL << 10; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("initial mmap"); + return; + } + p[0] = 1; + if (munmap((void *) p, len) != 0 || munmap((void *) p, len) != 0) { + FAIL("repeated munmap"); + return; + } + + /* This fault drains both retire records. Only the first still has region + * coverage, so only it may return VA to the allocator; the second must not + * hand the same extent out a second time. + */ + volatile uint8_t *bridge = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bridge == MAP_FAILED) { + FAIL("bridge mmap"); + return; + } + bridge[0] = 2; + + volatile uint8_t *a = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + volatile uint8_t *b = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (a == MAP_FAILED || b == MAP_FAILED || a == b) { + if (a != MAP_FAILED) + munmap((void *) a, len); + if (b != MAP_FAILED && b != a) + munmap((void *) b, len); + munmap((void *) bridge, len); + FAIL("duplicate extent allocation"); + return; + } + a[0] = 0x31; + b[0] = 0x42; + if (a[0] != 0x31 || b[0] != 0x42) { + FAIL("distinct mappings aliased"); + return; + } + munmap((void *) a, len); + munmap((void *) b, len); + munmap((void *) bridge, len); + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int state; + _Atomic int release; +} observer_arg_t; + +enum { + OBSERVER_STARTING, + OBSERVER_MAPPED, + OBSERVER_UNMAP_REQUESTED, + OBSERVER_UNMAPPED, + OBSERVER_MMAP_FAILED = -1, + OBSERVER_MUNMAP_FAILED = -2, +}; + +static void *observer_worker(void *opaque) +{ + observer_arg_t *arg = opaque; + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg->state, OBSERVER_MMAP_FAILED, + memory_order_release); + return NULL; + } + + atomic_store_explicit(&arg->ptr, (uintptr_t) p, memory_order_relaxed); + atomic_store_explicit(&arg->state, OBSERVER_MAPPED, memory_order_release); + while (atomic_load_explicit(&arg->state, memory_order_acquire) == + OBSERVER_MAPPED) + spin_hint(); + + if (munmap(p, 4096) != 0) { + atomic_store_explicit(&arg->state, OBSERVER_MUNMAP_FAILED, + memory_order_release); + return NULL; + } + atomic_store_explicit(&arg->state, OBSERVER_UNMAPPED, memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + return NULL; +} + +static int observer_wait_for_state(observer_arg_t *arg, int wanted) +{ + int state; + do { + state = atomic_load_explicit(&arg->state, memory_order_acquire); + spin_hint(); + } while (state >= 0 && state != wanted); + return state; +} + +static void observer_release(observer_arg_t *arg, pthread_t worker) +{ + atomic_store_explicit(&arg->release, 1, memory_order_release); + pthread_join(worker, NULL); +} + +static void test_msync_observes_fastpath(void) +{ + TEST("msync observes sibling fast mmap/munmap"); + observer_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, observer_worker, &arg) != 0) { + FAIL("pthread_create"); + return; + } + if (observer_wait_for_state(&arg, OBSERVER_MAPPED) != OBSERVER_MAPPED) { + pthread_join(worker, NULL); + FAIL("worker mmap"); + return; + } + + void *p = (void *) atomic_load_explicit(&arg.ptr, memory_order_relaxed); + bool mapped_visible = msync(p, 4096, MS_SYNC) == 0; + atomic_store_explicit(&arg.state, OBSERVER_UNMAP_REQUESTED, + memory_order_release); + int state = observer_wait_for_state(&arg, OBSERVER_UNMAPPED); + errno = 0; + bool unmapped_visible = state == OBSERVER_UNMAPPED && + msync(p, 4096, MS_SYNC) == -1 && errno == ENOMEM; + + observer_release(&arg, worker); + if (!mapped_visible || !unmapped_visible) { + FAIL("msync used stale fast-path VMA metadata"); + return; + } + PASS(); +} + +static bool proc_paths_contain(uintptr_t address, bool expected) +{ + char pid_maps[64], pid_smaps[64]; + snprintf(pid_maps, sizeof(pid_maps), "/proc/%ld/maps", (long) getpid()); + snprintf(pid_smaps, sizeof(pid_smaps), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = { + "/proc/self/maps", + pid_maps, + "/proc/self/smaps", + pid_smaps, + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + uintptr_t lo = 0, hi = 0; + int found = proc_extent_for(paths[i], address, &lo, &hi); + if (found < 0 || (found == 1) != expected) + return false; + } + return true; +} + +static void test_proc_observes_fastpath(void) +{ + TEST("maps/smaps observe sibling fast mmap/munmap"); + observer_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, observer_worker, &arg) != 0) { + FAIL("pthread_create"); + return; + } + if (observer_wait_for_state(&arg, OBSERVER_MAPPED) != OBSERVER_MAPPED) { + pthread_join(worker, NULL); + FAIL("worker mmap"); + return; + } + + uintptr_t p = atomic_load_explicit(&arg.ptr, memory_order_relaxed); + bool mapped_visible = proc_paths_contain(p, true); + atomic_store_explicit(&arg.state, OBSERVER_UNMAP_REQUESTED, + memory_order_release); + int state = observer_wait_for_state(&arg, OBSERVER_UNMAPPED); + bool unmapped_visible = + state == OBSERVER_UNMAPPED && proc_paths_contain(p, false); + + observer_release(&arg, worker); + if (!mapped_visible || !unmapped_visible) { + FAIL("maps/smaps used stale fast-path VMA metadata"); + return; + } + PASS(); +} + +typedef struct { + _Atomic int ready; + _Atomic int go; + _Atomic int stop; + _Atomic int failed; + _Atomic unsigned churned; +} wx_churn_arg_t; + +static void *wx_churn_worker(void *opaque) +{ + wx_churn_arg_t *arg = opaque; + atomic_fetch_add_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + + while (!atomic_load_explicit(&arg->stop, memory_order_acquire)) { + volatile uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg->failed, 1, memory_order_release); + break; + } + p[0] = 0x5a; + if (munmap((void *) p, 4096) != 0) { + atomic_store_explicit(&arg->failed, 1, memory_order_release); + break; + } + atomic_fetch_add_explicit(&arg->churned, 1, memory_order_release); + } + return NULL; +} + +static void test_wx_toggle_during_fastpath_churn(void) +{ + TEST("W^X toggle during sibling fast mmap churn"); + enum { NTHREADS = 4, NCYCLES = 256 }; + pthread_t workers[NTHREADS]; + wx_churn_arg_t arg = {0}; + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&workers[made], NULL, wx_churn_worker, &arg) != 0) + break; + } + if (made != NTHREADS) { + atomic_store_explicit(&arg.go, 1, memory_order_release); + atomic_store_explicit(&arg.stop, 1, memory_order_release); + for (int i = 0; i < made; i++) + pthread_join(workers[i], NULL); + FAIL("pthread_create"); + return; + } + + while (atomic_load_explicit(&arg.ready, memory_order_acquire) != NTHREADS) + spin_hint(); + + uint32_t *code = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok = code != MAP_FAILED; + if (ok) { + code[0] = 0x52800020u; /* mov w0, #1 */ + code[1] = 0xd65f03c0u; /* ret */ + __builtin___clear_cache((char *) code, (char *) (code + 2)); + uint32_t (*fn)(void) = (uint32_t (*)(void)) code; + ok = mprotect(code, 4096, PROT_READ | PROT_WRITE | PROT_EXEC) == 0 && + fn() == 1; + + atomic_store_explicit(&arg.go, 1, memory_order_release); + while (!atomic_load_explicit(&arg.failed, memory_order_acquire) && + atomic_load_explicit(&arg.churned, memory_order_acquire) < 64) + spin_hint(); + + for (uint32_t i = 0; i < NCYCLES && ok; i++) { + uint32_t imm = (i % 0xfffu) + 1; + code[0] = 0x52800000u | (imm << 5); + code[1] = 0xd65f03c0u; + __builtin___clear_cache((char *) code, (char *) (code + 2)); + if (fn() != imm) + ok = false; + } + } else { + atomic_store_explicit(&arg.go, 1, memory_order_release); + } + + atomic_store_explicit(&arg.stop, 1, memory_order_release); + for (int i = 0; i < NTHREADS; i++) + pthread_join(workers[i], NULL); + if (code != MAP_FAILED) + munmap(code, 4096); + + if (!ok || atomic_load_explicit(&arg.failed, memory_order_acquire) || + atomic_load_explicit(&arg.churned, memory_order_acquire) < 64) { + FAIL("W^X toggle or mmap churn failed"); + return; + } + PASS(); +} + +typedef struct { + int iterations; + _Atomic int *failed; +} storm_arg_t; + +static void *storm_worker(void *opaque) +{ + storm_arg_t *arg = opaque; + for (int i = 0; i < arg->iterations; i++) { + size_t len = (size_t) ((i & 7) + 1) * 4096; + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + p[0] = (uint8_t) i; + p[len - 1] = (uint8_t) (i ^ 0x5a); + if (munmap(p, len) != 0) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + } + return NULL; +} + +static void test_mt_storm_and_fork_exec(void) +{ + TEST("multi-vCPU mmap storm with fork+exec revocation"); + enum { NTHREADS = 8 }; + pthread_t threads[NTHREADS]; + _Atomic int failed = 0; + storm_arg_t arg = {.iterations = 400, .failed = &failed}; + + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&threads[made], NULL, storm_worker, &arg) != 0) { + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + break; + } + } + + pid_t pid = fork(); + if (pid == 0) { + char *const argv[] = {(char *) "/proc/self/exe", NULL}; + char *const envp[] = {(char *) "ELFUSE_FASTPATH_EXEC_CHILD=1", NULL}; + execve(argv[0], argv, envp); + _exit(111); + } + if (pid < 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + + for (int i = 0; i < made; i++) + pthread_join(threads[i], NULL); + + if (pid > 0) { + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + } + + /* The parent's arenas were revoked for the fork snapshot. This pair makes + * the first call take the generation fallback and verifies service resumes. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + else { + p[0] = 7; + munmap(p, 4096); + } + + if (__atomic_load_n(&failed, __ATOMIC_RELAXED)) { + FAIL("storm/fork/exec worker failure"); + return; + } + PASS(); +} + +static int stats_stream(size_t len, int iterations, bool release_each) +{ + for (int i = 0; i < iterations; i++) { + void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (release_each && munmap(p, len) != 0) + return 1; + } + return 0; +} + +static int stats_mixed_churn(void) +{ + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + for (int i = 0; i < 160; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (p[0] != 0 || p[len - 1] != 0) + return 1; + p[0] = 1; + p[len - 1] = 2; + if (munmap((void *) p, len) != 0) + return 1; + } + return 0; +} + +static int stats_large_materialized(void) +{ + const size_t len = 32ULL << 30; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 1; + return munmap((void *) p, len) != 0; +} + +static int stats_fork_no_topup(void) +{ + /* Leave exactly half of the initial 64 MiB arena free. The 32 MiB + * registration becomes the low-water mark, so the ordinary mmap lock + * acquire would speculatively refill immediately before fork revokes all + * arenas. + */ + void *p = mmap(NULL, 32ULL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + + pid_t pid = fork(); + if (pid == 0) + _exit(0); + if (pid < 0) + return 1; + + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + return 1; + return 0; +} + +static int stats_prefix_hint(void) +{ + void *p = mmap(NULL, 65536, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + void *q = mmap(p, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (q == MAP_FAILED || q == p) + return 1; + void *r = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (r == MAP_FAILED) + return 1; + return munmap(r, 4096) != 0 || munmap(q, 4096) != 0 || + munmap(p, 65536) != 0; +} + +static volatile sig_atomic_t altstack_signal_seen; + +static void altstack_signal_handler(int signum) +{ + altstack_signal_seen = signum; +} + +static int stats_signal_altstack(bool use_altstack) +{ + size_t size = 16ULL << 20; + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + stack_t stack = {.ss_sp = p, .ss_size = size}; + if (sigaltstack(&stack, NULL) != 0) + return 1; + struct sigaction action = {0}; + sigemptyset(&action.sa_mask); + action.sa_handler = SIG_IGN; + if (sigaction(SIGUSR1, &action, NULL) != 0 || raise(SIGUSR1) != 0) + return 1; + action.sa_handler = altstack_signal_handler; + action.sa_flags = use_altstack ? SA_ONSTACK : 0; + if (sigaction(SIGUSR1, &action, NULL) != 0 || raise(SIGUSR1) != 0 || + altstack_signal_seen != SIGUSR1) + return 1; + stack.ss_flags = SS_DISABLE; + if (sigaltstack(&stack, NULL) != 0) + return 1; + return munmap(p, size) != 0; +} + +static int stats_invalid_futex(void) +{ + uint32_t *words = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (words == MAP_FAILED) + return 1; + + long rc = raw_syscall6(__NR_futex, (long) words, 127, 0, 0, 0, 0); + if (rc != -38) + return 1; + rc = raw_syscall6(__NR_futex, (long) words, FUTEX_WAIT_BITSET, 0, 0, 0, 0); + if (rc != -22) + return 1; + rc = raw_syscall6(__NR_futex, (long) words, FUTEX_CMP_REQUEUE, 0, 0, + (long) ((uint8_t *) words + 1), 0); + if (rc != -22) + return 1; + rc = raw_syscall6(__NR_futex, (long) words, FUTEX_WAKE_OP, 0, 0, + (long) (words + 1), 5L << 28); + if (rc != -38) + return 1; + + return munmap(words, 4096) != 0; +} + +static int run_observer_semantics(void) +{ + test_msync_observes_fastpath(); + test_wx_toggle_during_fastpath_churn(); + test_proc_observes_fastpath(); + return fails != 0; +} + +static int run_stats_case(const char *name) +{ + if (strcmp(name, "prefix-hint") == 0) + return stats_prefix_hint(); + if (strcmp(name, "unused-altstack") == 0) + return stats_signal_altstack(false); + if (strcmp(name, "used-altstack") == 0) + return stats_signal_altstack(true); + if (strcmp(name, "observer-semantics") == 0) + return run_observer_semantics(); + if (strcmp(name, "ring-full") == 0) + return stats_stream(64ULL << 10, 40, false); + if (strcmp(name, "np2-10m") == 0) + return stats_stream(10ULL << 20, 96, false); + if (strcmp(name, "np2-48m") == 0) + return stats_stream(48ULL << 20, 48, false); + if (strcmp(name, "np2-100m") == 0) + return stats_stream(100ULL << 20, 30, false); + if (strcmp(name, "escalation") == 0) { + if (stats_stream(10ULL << 20, 48, false) != 0) + return 1; + return stats_stream(512ULL << 20, 3, false); + } + if (strcmp(name, "giant-guard") == 0) { + if (stats_stream(10ULL << 20, 32, false) != 0) + return 1; + for (int i = 0; i < 6; i++) { + if (stats_stream(2ULL << 30, 1, false) != 0 || + stats_stream(10ULL << 20, 1, false) != 0) + return 1; + } + return 0; + } + if (strcmp(name, "adaptive-small") == 0) + return stats_stream(64ULL << 10, 1100, false); + if (strcmp(name, "adaptive-retention") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0) + return 1; + + /* The first request selects a 16GiB arena, the next 32 consume it, and + * the last forces a capacity rollover that must retain the target. + */ + return stats_stream(500ULL << 20, 34, false); + } + if (strcmp(name, "adaptive-rewind-growth") == 0) + + /* The first 64MiB arena holds eight 8MiB mappings. The ninth mmap takes + * the capacity fallback after the matching munmaps let host drain + * rewind the arena; refill must grow it to the 32-entry target instead + * of retaining an arena that will miss every eight calls. + */ + return stats_stream(8ULL << 20, 41, true); + if (strcmp(name, "recycle") == 0) + return stats_stream(64ULL << 10, 6000, true); + if (strcmp(name, "mixed-churn") == 0) + return stats_mixed_churn(); + if (strcmp(name, "large-materialized") == 0) + return stats_large_materialized(); + if (strcmp(name, "fork-no-topup") == 0) + return stats_fork_no_topup(); + if (strcmp(name, "invalid-futex") == 0) + return stats_invalid_futex(); + return 2; +} + +enum { + METADATA_WORKERS = 4, + METADATA_MAPS = 2048, + METADATA_LENGTH = 2 * 1024 * 1024 + 4096 +}; +static void *metadata_maps[METADATA_WORKERS][METADATA_MAPS]; +static pthread_barrier_t metadata_barrier; +static _Atomic int metadata_full; + +static void *metadata_worker(void *arg) +{ + unsigned slot = (unsigned) (uintptr_t) arg; + unsigned count = 0; + int failed = 0; + pthread_barrier_wait(&metadata_barrier); + for (; count < METADATA_MAPS; count++) { + void *p = mmap(NULL, METADATA_LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + if (errno != ENOMEM) { + fprintf(stderr, "metadata mmap: errno=%d\n", errno); + failed = 1; + } else + atomic_store_explicit(&metadata_full, 1, memory_order_relaxed); + break; + } + metadata_maps[slot][count] = p; + } + + /* Keep every successful mapping live until all producers exhaust capacity. + * First touch follows a host exit that drains their publication rings. + */ + (void) fcntl(-1, F_GETFD); + pthread_barrier_wait(&metadata_barrier); + for (unsigned i = 0; i < count; i++) { + volatile uint8_t *p = metadata_maps[slot][i]; + if (*p != 0) + failed = 1; + *p = 0xa5; + if (*p != 0xa5) + failed = 1; + } + pthread_barrier_wait(&metadata_barrier); + for (unsigned i = 0; i < count; i++) + if (munmap(metadata_maps[slot][i], METADATA_LENGTH) != 0) + failed = 1; + return (void *) (uintptr_t) failed; +} + +static int metadata_exhaustion(void) +{ + pthread_t workers[METADATA_WORKERS]; + if (pthread_barrier_init(&metadata_barrier, NULL, METADATA_WORKERS) != 0) + return 1; + for (unsigned i = 0; i < METADATA_WORKERS; i++) { + int rc = pthread_create(&workers[i], NULL, metadata_worker, + (void *) (uintptr_t) i); + if (rc != 0) { + fprintf(stderr, "metadata pthread_create: %d\n", rc); + _exit(1); + } + } + int failed = 0; + for (unsigned i = 0; i < METADATA_WORKERS; i++) { + void *result; + if (pthread_join(workers[i], &result) != 0 || result != NULL) + failed = 1; + } + pthread_barrier_destroy(&metadata_barrier); + if (!atomic_load_explicit(&metadata_full, memory_order_relaxed)) { + fprintf(stderr, "metadata capacity not exhausted\n"); + failed = 1; + } + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + *(volatile uint8_t *) p = 0x5a; + return munmap(p, 4096) != 0 || failed; +} + +static void test_metadata_exhaustion(void) +{ + TEST("metadata credits under contention"); + pid_t pid = fork(); + if (pid == 0) + _exit(metadata_exhaustion()); + int status = 0; + EXPECT_TRUE(pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && WEXITSTATUS(status) == 0, + "metadata exhaustion or recovery failed"); +} + +int main(int argc, char **argv) +{ + if (argc == 2 && strcmp(argv[1], "--metadata-exhaustion") == 0) + return metadata_exhaustion(); + if (argc == 2 && strcmp(argv[1], "--observer-semantics") == 0) + return run_observer_semantics(); + if (argc == 2 && strncmp(argv[1], "--stats-", 8) == 0) + return run_stats_case(argv[1] + 8); + + if (getenv("ELFUSE_FASTPATH_EXEC_CHILD")) { + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 0xa5; + return p[0] == 0xa5 ? 0 : 1; + } + + test_metadata_exhaustion(); + test_fidelity(); + test_file_mmap_avoids_arena_hole(MAP_PRIVATE); + test_file_mmap_avoids_arena_hole(MAP_SHARED); + test_exhaustion_fallback(); + test_l3_range_tlbi(); + test_large_l2_range_tlbi(); + test_cross_vcpu_handoff(); + test_mixed_size_recycled_va_reads_zero(); + test_repeated_munmap_does_not_alias(); + run_observer_semantics(); + test_mt_storm_and_fork_exec(); + + printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, + fails ? "FAIL" : "PASS"); + return fails ? 1 : 0; +} diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c new file mode 100644 index 00000000..baeaf349 --- /dev/null +++ b/tests/test-mmap-lazy.c @@ -0,0 +1,1178 @@ +/* + * Lazy anonymous mmap regression tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Private anonymous mappings defer page-table creation and zeroing to first + * touch. These tests pin down the guest-visible contract of that laziness: huge + * reservations succeed and read as zeros, address reuse never leaks stale + * bytes, host-side syscall access (read/write/futex) works on memory the guest + * never touched, PROT_NONE stays a faulting reservation, data survives + * PROT_NONE round trips and fork, and concurrent first touch from multiple + * threads never loses a write to the deferred zeroing. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#define BLOCK_2MIB (2ULL << 20) + +#ifndef FUTEX_WAIT +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#endif + +/* Largest plain anonymous RW mapping the kernel grants. On elfuse the lazy path + * must take this well past physical memory; on real Linux the result depends on + * the overcommit heuristic, so the tests only require >= 1 GiB and probe + * downward. + */ +static void *map_largest(size_t *out_size) +{ + static const size_t sizes[] = { + 64ULL << 30, + 16ULL << 30, + 4ULL << 30, + 1ULL << 30, + }; + for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + void *p = mmap(NULL, sizes[i], PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + *out_size = sizes[i]; + return p; + } + } + return NULL; +} + +static void test_huge_sparse(void) +{ + TEST("huge mmap + sparse touch"); + size_t size = 0; + volatile uint8_t *p = map_largest(&size); + if (!p || size < (1ULL << 30)) { + FAIL("no >=1GiB anonymous mapping granted"); + return; + } + + /* Sparse probes: start, one per size/8 stride, last page. All must read + * zero and accept writes. + */ + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != 0) { + FAIL("fresh mapping reads nonzero"); + munmap((void *) p, size); + return; + } + p[off] = (uint8_t) (i + 1); + } + if (p[size - 1] != 0) { + FAIL("last page reads nonzero"); + munmap((void *) p, size); + return; + } + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != (uint8_t) (i + 1)) { + FAIL("sparse write lost"); + munmap((void *) p, size); + return; + } + } + if (munmap((void *) p, size) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +static void test_zero_reuse(void) +{ + TEST("address reuse reads zero"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xa5, size); + munmap(p, size); + uint8_t *q = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + + /* The allocator typically reuses the freed range; either way no byte may be + * nonzero. Check one page per 2MiB block plus both ends. + */ + for (size_t off = 0; off < size; off += 4096) { + if (q[off] != 0) { + FAIL("stale data after reuse"); + munmap(q, size); + return; + } + } + munmap(q, size); + PASS(); +} + +static void test_hinted_tail_zero(void) +{ + TEST("hinted tail mmap reads zero"); + size_t size = 2ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap base"); + return; + } + p[0] = 0x3c; + p[size - 1] = 0xc3; + + uint8_t *hint = p + size; + uint8_t *q = mmap(hint, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap hint"); + munmap(p, size); + return; + } + if (q[0] != 0 || q[size - 1] != 0 || p[0] != 0x3c || p[size - 1] != 0xc3) { + FAIL("hinted tail leaked stale bytes or clobbered neighbor"); + munmap(q, size); + munmap(p, size); + return; + } + munmap(q, size); + munmap(p, size); + PASS(); +} + +static void test_partial_block_reuse(void) +{ + TEST("partial-block reuse preserves neighbor"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[17] = 0xa5; + p[half + 17] = 0x5a; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED || q != p) { + FAIL("freed half was not reused at hint"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + if (q[17] != 0 || q[half - 1] != 0 || p[half + 17] != 0x5a) { + FAIL("partial zero clobbered neighbor or leaked stale data"); + munmap(q, half); + munmap(p + half, half); + return; + } + munmap(q, half); + munmap(p + half, half); + PASS(); +} + +static uint8_t *map_clean_block_reuse(void) +{ + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return MAP_FAILED; + memset(p, 0xcc, BLOCK_2MIB); + if (munmap(p, BLOCK_2MIB) != 0) { + munmap(p, BLOCK_2MIB); + return MAP_FAILED; + } + + uint8_t *q = mmap(p, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (q != p) { + if (q != MAP_FAILED) + munmap(q, BLOCK_2MIB); + return MAP_FAILED; + } + return q; +} + +static void test_file_overlay_reuse(void) +{ + TEST("file overlay teardown then lazy reuse"); + char path[] = "/tmp/elfuse-dirty-map.XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + if (ftruncate(fd, BLOCK_2MIB) != 0) { + FAIL("ftruncate"); + close(fd); + return; + } + uint8_t first = 0xa7, last = 0x5c; + if (pwrite(fd, &first, 1, 17) != 1 || + pwrite(fd, &last, 1, BLOCK_2MIB - 1) != 1) { + FAIL("pwrite"); + close(fd); + return; + } + + uint8_t *reserve = mmap(NULL, 2 * BLOCK_2MIB, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserve == MAP_FAILED) { + FAIL("reserve"); + close(fd); + return; + } + uintptr_t aligned = + ((uintptr_t) reserve + BLOCK_2MIB - 1) & ~(BLOCK_2MIB - 1); + munmap(reserve, 2 * BLOCK_2MIB); + uint8_t *target = (uint8_t *) aligned; + + uint8_t *file = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (file != target || file[17] != first || file[BLOCK_2MIB - 1] != last) { + FAIL("file mmap"); + if (file != MAP_FAILED) + munmap(file, BLOCK_2MIB); + close(fd); + return; + } + file[BLOCK_2MIB / 2] = 0xe1; + if (munmap(file, BLOCK_2MIB) != 0) { + FAIL("file munmap"); + close(fd); + return; + } + close(fd); + + uint8_t *anon = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon != target || anon[17] != 0 || anon[BLOCK_2MIB / 2] != 0 || + anon[BLOCK_2MIB - 1] != 0) { + FAIL("stale file bytes after lazy reuse"); + if (anon != MAP_FAILED) + munmap(anon, BLOCK_2MIB); + return; + } + munmap(anon, BLOCK_2MIB); + PASS(); +} + +static void test_read_into_lazy(void) +{ + TEST("read() into untouched mapping"); + size_t size = 6ULL << 20; + uint8_t *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (buf == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + static const char msg[] = "lazy-host-access-payload"; + /* Unaligned target crossing into the mapping's third 2MiB block. */ + size_t off = (4ULL << 20) + 123; + if (write(fds[1], msg, sizeof(msg)) != (ssize_t) sizeof(msg) || + read(fds[0], buf + off, sizeof(msg)) != (ssize_t) sizeof(msg)) { + FAIL("pipe copy through untouched buffer"); + goto out; + } + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload corrupted"); + goto out; + } + + /* A guest touch elsewhere in the same 2MiB block must not re-zero the + * host-written payload (deferred-zeroing idempotence). + */ + buf[(4ULL << 20) + 64 * 1024] = 7; + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload clobbered by later fault in same block"); + goto out; + } + /* Untouched parts of the mapping still read zero. */ + for (size_t i = 0; i < 4096; i++) { + if (buf[i] != 0) { + FAIL("nonzero byte in untouched block"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(buf, size); +} + +static void test_write_from_lazy(void) +{ + TEST("write() from untouched mapping"); + size_t size = 2ULL << 20; + uint8_t *src = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (src == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + uint8_t back[512]; + memset(back, 0xff, sizeof(back)); + if (write(fds[1], src + 4096, sizeof(back)) != (ssize_t) sizeof(back) || + read(fds[0], back, sizeof(back)) != (ssize_t) sizeof(back)) { + FAIL("pipe copy from untouched buffer"); + goto out; + } + for (size_t i = 0; i < sizeof(back); i++) { + if (back[i] != 0) { + FAIL("untouched buffer sent nonzero bytes"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(src, size); +} + +static void test_prot_none_roundtrip(void) +{ + TEST("mprotect NONE round trip keeps data"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x5c, 8192); + p[size - 1] = 0x77; + if (mprotect(p, size, PROT_NONE) != 0 || + mprotect(p, size, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect"); + munmap(p, size); + return; + } + if (p[0] != 0x5c || p[8191] != 0x5c || p[size - 1] != 0x77 || + p[16384] != 0) { + FAIL("data lost or stale bytes after round trip"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_reserve_commit(void) +{ + TEST("PROT_NONE reserve + mprotect commit"); + size_t size = 1ULL << 30; + uint8_t *p = mmap(NULL, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) { + FAIL("reserve"); + return; + } + uint8_t *slab = p + (512ULL << 20); + if (mprotect(slab, 8ULL << 20, PROT_READ | PROT_WRITE) != 0) { + FAIL("commit"); + munmap(p, size); + return; + } + for (size_t off = 0; off < (8ULL << 20); off += 4096) { + if (slab[off] != 0) { + FAIL("committed slab reads nonzero"); + munmap(p, size); + return; + } + } + slab[0] = 1; + slab[(8ULL << 20) - 1] = 2; + if (slab[0] != 1 || slab[(8ULL << 20) - 1] != 2) { + FAIL("committed slab write lost"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static void test_prot_none_faults(void) +{ + TEST("PROT_NONE|NORESERVE still faults"); + size_t size = 16ULL << 20; + volatile uint8_t *p = + mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + struct sigaction sa = {0}, old_sa; + sa.sa_handler = segv_handler; + sigaction(SIGSEGV, &sa, &old_sa); + int faulted = 0; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) p[BLOCK_2MIB + 5]; + } else { + faulted = 1; + } + sigaction(SIGSEGV, &old_sa, NULL); + munmap((void *) p, size); + + /* A lazy materializer that ignores prot would silently hand the guest a + * readable zero page here instead of SIGSEGV. + */ + EXPECT_TRUE(faulted, "read from PROT_NONE reservation did not fault"); +} + +static void test_fork_lazy(void) +{ + TEST("fork with lazy and clean-block reused mappings"); + size_t size = 8ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uint8_t *clean = map_clean_block_reuse(); + if (clean == MAP_FAILED) { + FAIL("clean-block reuse"); + munmap(p, size); + return; + } + + memset(p, 0x42, 4096); /* touch only block 0 */ + pid_t pid = fork(); + if (pid == 0) { + /* Child: inherited data intact, untouched block reads zero and is + * privately writable. + */ + if (p[0] != 0x42 || p[4095] != 0x42) + _exit(1); + if (p[4ULL << 20] != 0) + _exit(2); + p[4ULL << 20] = 0x99; + if (p[(4ULL << 20) + 1] != 0) + _exit(3); + if (clean[0] != 0 || clean[BLOCK_2MIB - 1] != 0) + _exit(4); + clean[123] = 0x77; + if (clean[124] != 0) + _exit(5); + _exit(0); + } + int st = 0; + bool ok = pid >= 0 && waitpid(pid, &st, 0) == pid && WIFEXITED(st) && + WEXITSTATUS(st) == 0; + /* Parent: child's private write must not leak back. */ + ok = ok && p[4ULL << 20] == 0 && clean[123] == 0; + munmap(clean, BLOCK_2MIB); + munmap(p, size); + if (!ok) { + FAIL("fork changed lazy or clean-block reused memory"); + return; + } + PASS(); +} + +static void test_futex_untouched(void) +{ + TEST("futex on untouched mapping"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + uint32_t *word = (uint32_t *) (p + (2ULL << 20) + 256); + /* WAKE on never-touched memory: no waiters, must not fault. */ + long r = syscall(SYS_futex, word, FUTEX_WAKE, 1, NULL, NULL, 0); + if (r != 0) { + FAIL("FUTEX_WAKE on untouched word"); + munmap(p, size); + return; + } + /* WAIT with expected=1: the word reads as zero, so EAGAIN. */ + r = syscall(SYS_futex, word, FUTEX_WAIT, 1, NULL, NULL, 0); + if (!(r == -1 && errno == EAGAIN)) { + FAIL("FUTEX_WAIT did not read zero from untouched word"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +/* Concurrent first touch: every thread writes its own slot in the same fresh + * 2MiB block, racing the deferred zeroing. A materializer that re-zeros an + * already-populated block loses some slots. + */ +#define MT_THREADS 4 +#define MT_ITERS 64 + +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t start; + pthread_cond_t done; + pthread_barrier_t touch; + uint8_t *base; + unsigned generation; + int completed; + bool stop; +} mt_state_t; + +typedef struct { + mt_state_t *state; + int idx; +} mt_arg_t; + +static bool mt_state_init(mt_state_t *state) +{ + memset(state, 0, sizeof(*state)); + if (pthread_mutex_init(&state->mutex, NULL) != 0) + return false; + if (pthread_cond_init(&state->start, NULL) != 0) { + pthread_mutex_destroy(&state->mutex); + return false; + } + if (pthread_cond_init(&state->done, NULL) != 0) { + pthread_cond_destroy(&state->start); + pthread_mutex_destroy(&state->mutex); + return false; + } + if (pthread_barrier_init(&state->touch, NULL, MT_THREADS) != 0) { + pthread_cond_destroy(&state->done); + pthread_cond_destroy(&state->start); + pthread_mutex_destroy(&state->mutex); + return false; + } + return true; +} + +static void mt_state_destroy(mt_state_t *state) +{ + pthread_barrier_destroy(&state->touch); + pthread_cond_destroy(&state->done); + pthread_cond_destroy(&state->start); + pthread_mutex_destroy(&state->mutex); +} + +static void *mt_touch(void *argp) +{ + mt_arg_t *a = argp; + mt_state_t *state = a->state; + unsigned seen = 0; + + pthread_mutex_lock(&state->mutex); + for (;;) { + while (!state->stop && seen == state->generation) + pthread_cond_wait(&state->start, &state->mutex); + if (state->stop) + break; + + uint8_t *base = state->base; + seen = state->generation; + pthread_mutex_unlock(&state->mutex); + + pthread_barrier_wait(&state->touch); + base[a->idx * 64] = (uint8_t) (a->idx + 1); + /* Also touch a private block so several materializations race. */ + base[BLOCK_2MIB * (unsigned) (a->idx + 1) + 17] = + (uint8_t) (0x10 + a->idx); + + pthread_mutex_lock(&state->mutex); + state->completed++; + if (state->completed == MT_THREADS) + pthread_cond_signal(&state->done); + } + pthread_mutex_unlock(&state->mutex); + return NULL; +} + +static void test_mt_first_touch(void) +{ + TEST("concurrent first touch"); + mt_state_t state; + if (!mt_state_init(&state)) { + FAIL("worker synchronization setup"); + return; + } + pthread_t th[MT_THREADS]; + mt_arg_t args[MT_THREADS]; + int created = 0; + bool ok = true; + + /* Reuse workers so all 64 rounds stress materialization rather than glibc + * thread-stack allocation and teardown. + */ + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (mt_arg_t) {&state, i}; + if (pthread_create(&th[i], NULL, mt_touch, &args[i]) != 0) { + ok = false; + break; + } + created++; + } + + for (int iter = 0; ok && iter < MT_ITERS; iter++) { + size_t size = BLOCK_2MIB * (MT_THREADS + 2); + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = false; + break; + } + + pthread_mutex_lock(&state.mutex); + state.base = p; + state.completed = 0; + state.generation++; + pthread_cond_broadcast(&state.start); + while (state.completed != MT_THREADS) + pthread_cond_wait(&state.done, &state.mutex); + pthread_mutex_unlock(&state.mutex); + + for (int i = 0; i < MT_THREADS; i++) { + if (p[i * 64] != (uint8_t) (i + 1) || + p[BLOCK_2MIB * (unsigned) (i + 1) + 17] != + (uint8_t) (0x10 + i)) { + ok = false; + break; + } + } + munmap(p, size); + } + + pthread_mutex_lock(&state.mutex); + state.stop = true; + pthread_cond_broadcast(&state.start); + pthread_mutex_unlock(&state.mutex); + for (int i = 0; i < created; i++) + pthread_join(th[i], NULL); + mt_state_destroy(&state); + + if (!ok) { + FAIL("setup failed or a write was lost to materialization"); + return; + } + PASS(); +} + +typedef struct { + uint8_t *base; + size_t half; + int idx; + pthread_barrier_t *barrier; + int *error; + _Atomic int *start; +} claim_race_arg_t; + +static bool claim_race_start(claim_race_arg_t *a) +{ + int start; + while ((start = atomic_load_explicit(a->start, memory_order_acquire)) == 0) + sched_yield(); + if (start < 0) + return false; + pthread_barrier_wait(a->barrier); + return true; +} + +static void *claim_race_touch(void *argp) +{ + claim_race_arg_t *a = argp; + if (!claim_race_start(a)) + return NULL; + a->base[64 * (unsigned) a->idx] = (uint8_t) (a->idx + 1); + return NULL; +} + +static void *claim_race_mutate(void *argp) +{ + claim_race_arg_t *a = argp; + uint8_t *neighbor = a->base + a->half; + uint8_t *hole = neighbor + 4096; + if (!claim_race_start(a)) + return NULL; + for (int i = 0; i < 16; i++) { + if (mprotect(neighbor, a->half, PROT_NONE) != 0 || + mprotect(neighbor, a->half, PROT_READ) != 0 || + munmap(hole, 4096) != 0) { + *a->error = 1; + return NULL; + } + void *r = mmap(hole, 4096, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (r != hole) { + *a->error = 1; + return NULL; + } + } + return NULL; +} + +static void test_claim_mutation_race(void) +{ + TEST("first-touch claim vs adjacent mutations"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[half + 17] = 0x6d; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q != p) { + FAIL("reuse"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + + pthread_barrier_t barrier; + if (pthread_barrier_init(&barrier, NULL, MT_THREADS + 1) != 0) { + FAIL("barrier setup"); + munmap(p, BLOCK_2MIB); + return; + } + pthread_t workers[MT_THREADS + 1]; + claim_race_arg_t args[MT_THREADS + 1]; + int error = 0; + int created = 0; + _Atomic int start = 0; + for (int i = 0; i < MT_THREADS + 1; i++) { + args[i] = (claim_race_arg_t) {q, half, i, &barrier, &error, &start}; + int rc = pthread_create( + &workers[i], NULL, + i == MT_THREADS ? claim_race_mutate : claim_race_touch, &args[i]); + if (rc != 0) { + error = rc; + break; + } + created++; + } + + /* Release pairs with the worker acquire: no barrier entry until the full + * group exists, and partial groups exit before their arguments expire. + */ + atomic_store_explicit(&start, error ? -1 : 1, memory_order_release); + for (int i = 0; i < created; i++) + pthread_join(workers[i], NULL); + pthread_barrier_destroy(&barrier); + + if (created != MT_THREADS + 1) { + errno = error; + FAIL("pthread_create"); + munmap(p, BLOCK_2MIB); + return; + } + + for (int i = 0; i < MT_THREADS; i++) { + if (q[64 * (unsigned) i] != (uint8_t) (i + 1)) + error = 1; + } + if (p[half + 17] != 0x6d) + error = 1; + munmap(q, half); + munmap(p + half, half); + if (error) { + FAIL("claim/mutation race corrupted data"); + return; + } + PASS(); +} + +static void test_adjacent_region_extension(void) +{ + TEST("adjacent fast-mmap region extension"); + enum { N_PAGES = 64 }; + uint8_t *pages[N_PAGES]; + int allocated = 0; + bool ok = true; + + for (int i = 0; i < N_PAGES; i++) { + pages[i] = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages[i] == MAP_FAILED) { + ok = false; + break; + } + allocated++; + pages[i][0] = (uint8_t) (i + 1); + } + for (int i = 0; ok && i < allocated; i++) { + if (pages[i][0] != (uint8_t) (i + 1)) + ok = false; + } + for (int i = 0; i < allocated; i++) + munmap(pages[i], 4096); + + if (!ok) { + FAIL("adjacent lazy mappings did not materialize independently"); + return; + } + PASS(); +} + +static void test_large_retire_reuse(void) +{ + TEST("large multi-block retire preserves neighbors and fork zeroes"); + const size_t body_len = 96ULL << 20; + const size_t total_len = body_len + (6ULL << 20); + uint8_t *p = mmap(NULL, total_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uintptr_t aligned = + ((uintptr_t) p + BLOCK_2MIB * 2 - 1) & ~(uintptr_t) (BLOCK_2MIB - 1); + uint8_t *body = (uint8_t *) aligned; + size_t left_len = (size_t) (body - p); + size_t right_len = total_len - left_len - body_len; + if (left_len < BLOCK_2MIB || right_len < BLOCK_2MIB) { + FAIL("guard alignment"); + munmap(p, total_len); + return; + } + + memset(p, 0xa5, total_len); + if (munmap(body, body_len) != 0) { + FAIL("retire body"); + munmap(p, total_len); + return; + } + + /* MAP_FIXED is a metadata-reading slow path, so it must first drain the EL1 + * retirement. The 96 MiB dirty body spans many 2 MiB blocks, so this + * exercises retire-then-reuse at multi-block scale rather than the + * single-block case the smaller tests above already cover. + */ + uint8_t *q = mmap(body, body_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (q != body) { + FAIL("fixed reuse"); + munmap(p, left_len); + munmap(body + body_len, right_len); + return; + } + + bool ok = p[0] == 0xa5 && body[-1] == 0xa5 && body[body_len] == 0xa5 && + p[total_len - 1] == 0xa5; + for (size_t off = 0; ok && off < body_len; off += 4096) + ok = q[off] == 0; + + pid_t pid = -1; + int st = 0; + if (ok) + pid = fork(); + if (pid == 0) { + for (size_t off = 0; off < body_len; off += BLOCK_2MIB) { + if (q[off] != 0) + _exit(1); + } + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) + ok = false; + + munmap(p, left_len); + munmap(q, body_len); + munmap(body + body_len, right_len); + if (!ok) { + FAIL("reuse leaked data, clobbered a neighbor, or broke fork"); + return; + } + PASS(); +} + +static void test_mremap_zero_reuse(bool fixed, bool touched) +{ + TEST(fixed ? (touched ? "fixed mremap preserves mixed lazy source" + : "fixed mremap zeroes untouched reused source") + : (touched ? "moving mremap preserves mixed lazy source" + : "moving mremap zeroes untouched reused source")); + const size_t total_len = 10 * BLOCK_2MIB; + const size_t old_len = 8 * BLOCK_2MIB - 32768; + const size_t new_len = old_len + BLOCK_2MIB; + uint8_t *dest = MAP_FAILED; + if (fixed) { + dest = + mmap(NULL, new_len, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (dest == MAP_FAILED) { + FAIL("mmap destination"); + return; + } + } + uint8_t *base = mmap(NULL, total_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) { + FAIL("mmap source"); + if (fixed) + munmap(dest, new_len); + return; + } + + /* Live neighbors share both edge blocks and prevent in-place growth. */ + uintptr_t aligned = + ((uintptr_t) base + BLOCK_2MIB - 1) & ~(uintptr_t) (BLOCK_2MIB - 1); + uint8_t *source = (uint8_t *) aligned + 16384; + memset(base, 0xa5, total_len); + /* A permission split releases the fast arena's reservation of the hole. */ + if (mprotect(base, 4096, PROT_READ) != 0 || munmap(source, old_len) != 0) { + FAIL("split and unmap source"); + munmap(base, total_len); + if (fixed) + munmap(dest, new_len); + return; + } + /* MAP_FIXED eagerly zeroes its backing and would hide the lazy-copy bug. */ + uint8_t *reused = mmap(source, old_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reused != source) { + FAIL("reuse source"); + if (reused != MAP_FAILED) + munmap(reused, old_len); + munmap(base, total_len); + if (fixed) + munmap(dest, new_len); + return; + } + + const size_t marker = 2 * BLOCK_2MIB + 123; + if (touched) { + source[marker] = 0x5a; + source[old_len - 1] = 0x7e; + } + uint8_t *moved = mremap(source, old_len, new_len, + MREMAP_MAYMOVE | (fixed ? MREMAP_FIXED : 0), dest); + if (moved == MAP_FAILED) { + FAIL("mremap"); + munmap(base, total_len); + if (fixed) + munmap(dest, new_len); + return; + } + + bool ok = moved != source && (!fixed || moved == dest) && base[0] == 0xa5 && + source[-1] == 0xa5 && source[old_len] == 0xa5 && + base[total_len - 1] == 0xa5; + for (size_t off = 0; ok && off < new_len; off++) { + uint8_t expected = touched && off == marker ? 0x5a + : touched && off == old_len - 1 ? 0x7e + : 0; + ok = moved[off] == expected; + } + munmap(moved, new_len); + munmap(base, total_len); + EXPECT_TRUE(ok, "mremap leaked retired bytes or changed live data"); +} + +static void test_locked_syscall_buffers(void) +{ + const size_t len = 16 * BLOCK_2MIB; + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + TEST("locked syscall buffers setup"); + FAIL("mmap"); + return; + } + + TEST("timerfd untouched input/output"); + int fd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + int rc = fd < 0 + ? -1 + : timerfd_settime(fd, 0, (const struct itimerspec *) p, + (struct itimerspec *) (p + 2 * BLOCK_2MIB)); + EXPECT_TRUE(rc == 0, "timerfd could not copy lazy zero values"); + if (fd >= 0) { + TEST("timerfd input/output alias"); + struct itimerspec aliased = {.it_value = {.tv_sec = 60}}; + struct itimerspec armed = {0}; + rc = timerfd_settime(fd, 0, &aliased, &aliased); + EXPECT_TRUE(rc == 0 && aliased.it_value.tv_sec == 0 && + timerfd_gettime(fd, &armed) == 0 && + armed.it_value.tv_sec > 0, + "timerfd copied old output before staging new input"); + close(fd); + } + + TEST("sigprocmask untouched input/output"); + rc = (int) syscall(SYS_rt_sigprocmask, SIG_BLOCK, p + 4 * BLOCK_2MIB, + p + 6 * BLOCK_2MIB, 8); + EXPECT_TRUE(rc == 0, "sigprocmask could not copy lazy empty mask"); + + TEST("sigaction untouched input/output"); + rc = (int) syscall(SYS_rt_sigaction, SIGUSR2, p + 8 * BLOCK_2MIB, + p + 10 * BLOCK_2MIB, 8); + EXPECT_TRUE(rc == 0, "sigaction could not copy lazy default action"); + + TEST("getcwd untouched output"); + long n = syscall(SYS_getcwd, p + 12 * BLOCK_2MIB, 4096); + EXPECT_TRUE(n > 0 && p[12 * BLOCK_2MIB] == '/', + "getcwd could not fill lazy buffer"); + + TEST("getdents untouched output"); + fd = open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC); + n = fd < 0 ? -1 : syscall(SYS_getdents64, fd, p + 14 * BLOCK_2MIB, 65536); + EXPECT_TRUE(n > 0, "getdents could not fill lazy buffer"); + if (fd >= 0) + close(fd); + munmap(p, len); +} + +static void test_netlink_lazy_output(void) +{ + TEST("netlink untouched receive buffer"); + const size_t len = 2 * BLOCK_2MIB; + void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); + if (p == MAP_FAILED || fd < 0) { + FAIL("netlink setup"); + } else { + struct { + struct nlmsghdr hdr; + struct ifinfomsg msg; + } req = { + .hdr = {.nlmsg_len = sizeof(req), + .nlmsg_type = RTM_GETLINK, + .nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP, + .nlmsg_seq = 1}, + .msg = {.ifi_family = AF_UNSPEC}, + }; + struct sockaddr_nl dst = {.nl_family = AF_NETLINK}; + ssize_t sent = sendto(fd, &req, sizeof(req), 0, + (struct sockaddr *) &dst, sizeof(dst)); + ssize_t received = sent == sizeof(req) ? recv(fd, p, 8192, 0) : -1; + EXPECT_TRUE(received >= (ssize_t) sizeof(struct nlmsghdr), + "netlink could not fill lazy buffer"); + } + if (fd >= 0) + close(fd); + if (p != MAP_FAILED) + munmap(p, len); +} + +static void test_waitid_lazy_output(void) +{ + TEST("waitid untouched output and WNOWAIT"); + const size_t len = 2 * BLOCK_2MIB; + siginfo_t *si = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (si == MAP_FAILED) { + FAIL("mmap"); + return; + } + pid_t pid = fork(); + if (pid == 0) + _exit(23); + bool faulted = pid > 0 && + syscall(SYS_waitid, P_PID, pid, (void *) 8, + WEXITED | WNOWAIT, NULL) == -1 && + errno == EFAULT; + int rc = pid < 0 ? -1 : waitid(P_PID, pid, si, WEXITED | WNOWAIT); + bool ok = faulted && rc == 0 && si->si_pid == pid && si->si_status == 23; + if (pid > 0) { + int status = 0; + ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + WEXITSTATUS(status) == 23 && ok; + } + EXPECT_TRUE(ok, "waitid lost lazy output or consumed WNOWAIT status"); + munmap(si, len); +} + +int main(void) +{ + test_huge_sparse(); + test_zero_reuse(); + test_hinted_tail_zero(); + test_partial_block_reuse(); + test_file_overlay_reuse(); + test_read_into_lazy(); + test_write_from_lazy(); + test_prot_none_roundtrip(); + test_reserve_commit(); + test_prot_none_faults(); + test_fork_lazy(); + test_futex_untouched(); + test_mt_first_touch(); + test_claim_mutation_race(); + test_adjacent_region_extension(); + test_large_retire_reuse(); + test_mremap_zero_reuse(true, false); + test_mremap_zero_reuse(true, true); + test_mremap_zero_reuse(false, false); + test_mremap_zero_reuse(false, true); + test_locked_syscall_buffers(); + test_netlink_lazy_output(); + test_waitid_lazy_output(); + + SUMMARY("test-mmap-lazy"); + return fails ? 1 : 0; +} diff --git a/tests/test-mremap.c b/tests/test-mremap.c index dc97321b..8bdc3447 100644 --- a/tests/test-mremap.c +++ b/tests/test-mremap.c @@ -120,6 +120,106 @@ static void test_grow_maymove(void) munmap(q, 4096 * 4); } +static void test_grow_move_adjacent_fault(void) +{ + TEST("mremap fixed move keeps adjacent page unmapped"); + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *dest = + mmap(NULL, 4096 * 3, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (dest == MAP_FAILED) { + FAIL("mmap dest"); + munmap(p, 4096); + return; + } + if (munmap(dest + 8192, 4096) < 0) { + FAIL("munmap guard"); + munmap(dest, 4096 * 3); + munmap(p, 4096); + return; + } + ((char *) p)[0] = 0x5a; + + void *q = mremap(p, 4096, 8192, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (q == MAP_FAILED) { + FAIL("mremap move"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + if (q != dest || ((char *) q)[0] != 0x5a || ((char *) q)[4096] != 0) { + FAIL("mremap data"); + munmap(q, 8192); + return; + } + + pid_t pid = fork(); + if (pid == 0) { + volatile unsigned char value = *((volatile unsigned char *) q + 8192); + (void) value; + _exit(1); + } + int status = 0; + if (pid < 0 || waitpid(pid, &status, 0) != pid) { + FAIL("fork/wait"); + munmap(q, 8192); + return; + } + if ((WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV) || + (WIFEXITED(status) && WEXITSTATUS(status) == 139)) + PASS(); + else + FAIL("adjacent page became readable"); + munmap(q, 8192); +} + +static void test_fixed_preserves_neighbor_l3(void) +{ + TEST("mremap fixed preserves neighboring lazy PTEs"); + char *source = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (source == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *anchor = mmap(NULL, 8192, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anchor == MAP_FAILED) { + FAIL("mmap anchor"); + munmap(source, 4096); + return; + } + char *dest = anchor + 4096; + if (munmap(dest, 4096) < 0) { + FAIL("munmap dest"); + munmap(anchor, 8192); + munmap(source, 4096); + return; + } + source[0] = 0x11; + anchor[0] = 0x7e; + + void *moved = + mremap(source, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (moved == MAP_FAILED) { + FAIL("mremap fixed"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + if (moved != dest || dest[0] != 0x11 || anchor[0] != 0x7e) + FAIL("neighboring lazy page changed"); + else + PASS(); + + munmap(dest, 4096); + munmap(anchor, 4096); +} + /* Test 3: grow without MAYMOVE fails if blocked */ static void test_grow_no_maymove(void) @@ -403,6 +503,8 @@ int main(void) test_shrink(); test_grow_maymove(); + test_grow_move_adjacent_fault(); + test_fixed_preserves_neighbor_l3(); test_grow_no_maymove(); test_fixed(); test_same_size(); diff --git a/tests/test-pt-epoch-host.c b/tests/test-pt-epoch-host.c new file mode 100644 index 00000000..22b25534 --- /dev/null +++ b/tests/test-pt-epoch-host.c @@ -0,0 +1,80 @@ +/* + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * EL1 fast munmap clears descriptors without touching guest_t.pt_gen. A host + * thread's translation cache must drop an entry taken before that clear as soon + * as the EL1 PT epoch moves, before any host writer bumps pt_gen. + */ + +#include +#include +#include +#include + +#include "core/guest.h" +#include "core/mmap-fastpath.h" +#include "syscall/internal.h" +#include "test-harness.h" + +int passes, fails; + +static guest_t *g; +static const uint64_t block = 2ULL << 20; +static const uint64_t addr = 16ULL << 20; + +/* The host's view of el1_munmap: descriptors gone, pt_gen untouched, epoch + * advanced. A separate thread, so the main thread's cache entry survives the + * way one on a thread that never drains would. + */ +static void *retire_like_el1(void *arg) +{ + (void) arg; + _Atomic uint64_t *epoch = + (_Atomic uint64_t *) ((uint8_t *) g->host_base + g->shim_data_base + + SHIM_MMAP_PT_EPOCH_OFF); + mmap_lock_acquire(g); + uint64_t gen = atomic_load_explicit(&g->pt_gen, memory_order_acquire); + int rc = guest_invalidate_ptes(g, addr, addr + block); + atomic_store_explicit(&g->pt_gen, gen, memory_order_release); + atomic_fetch_add_explicit(epoch, 1, memory_order_release); + mmap_lock_release(); + return (void *) (intptr_t) rc; +} + +int main(void) +{ + g = calloc(1, sizeof(*g)); + if (!g || guest_init(g, 64ULL << 30, 0) != 0) + return 1; + if (!guest_build_page_tables(g, NULL, 0)) + return 1; + if (guest_region_add( + g, addr, addr + block, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL) < 0) + return 1; + + TEST("EL1 PT epoch retires a cached translation"); + uint64_t avail = 0; + void *before = guest_ptr_bound(g, addr, &avail, MEM_PERM_W, 1); + void *cached = guest_ptr_avail_nofault(g, addr, &avail, MEM_PERM_R); + pthread_t retirer; + void *rc = NULL; + if (!before || cached != before || + pthread_create(&retirer, NULL, retire_like_el1, NULL) != 0) + return 1; + pthread_join(retirer, &rc); + void *after = guest_ptr_avail_nofault(g, addr, &avail, MEM_PERM_R); + EXPECT_TRUE(rc == NULL && after == NULL, + "retired range still served from the host cache"); + + TEST("translation resumes after rematerialization"); + void *again = guest_ptr_bound(g, addr, &avail, MEM_PERM_W, 1); + EXPECT_TRUE(again == before, "rematerialized block moved or failed"); + + guest_destroy(g); + free(g); + SUMMARY("test-pt-epoch-host"); + return fails != 0; +} diff --git a/tests/test-shim-futex-fast.c b/tests/test-shim-futex-fast.c index c80c5278..185b1efe 100644 --- a/tests/test-shim-futex-fast.c +++ b/tests/test-shim-futex-fast.c @@ -152,6 +152,72 @@ static void test_tagged_address(void) expect(raw_futex_wait((int *) tagged, 1), -EFAULT, "tagged uaddr match"); } +static void test_lazy_wait_case(int op, int timed, uint32_t expected) +{ + const size_t span = 4ULL << 20; + uint8_t *p = mmap(NULL, 2 * span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + failures++; + perror("lazy futex mmap"); + return; + } + + /* Each call gets untouched word and timeout pages in separate blocks. */ + long word = (long) (p + span); + long timeout = timed ? (long) p : 0; + expect(raw_syscall6(__NR_futex, word, op | FUTEX_PRIVATE_FLAG, expected, + timeout, 0, FUTEX_BITSET_MATCH_ANY), + expected ? -EAGAIN : -ETIMEDOUT, + expected ? "lazy wait mismatch" : "zero lazy timeout"); + expect(munmap(p, 2 * span), 0, "lazy futex munmap"); +} + +static void test_lazy_wait_inputs(void) +{ + const int ops[] = {FUTEX_WAIT, FUTEX_WAIT_BITSET}; + for (unsigned i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + test_lazy_wait_case(ops[i], 0, 1); + test_lazy_wait_case(ops[i], 1, 1); + test_lazy_wait_case(ops[i], 1, 0); + } +} + +static void test_page_boundary(void) +{ + uint8_t *p = mmap(NULL, 2 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + failures++; + perror("boundary futex mmap"); + return; + } + if (mprotect(p + PAGE_SIZE, PAGE_SIZE, PROT_NONE) != 0) { + failures++; + perror("boundary futex mprotect"); + munmap(p, 2 * PAGE_SIZE); + return; + } + const int ops[] = {FUTEX_WAIT, FUTEX_WAIT_BITSET}; + for (unsigned i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + expect(raw_syscall6(__NR_futex, (long) (p + PAGE_SIZE - 4), + ops[i] | FUTEX_PRIVATE_FLAG, 1, 0, 0, + FUTEX_BITSET_MATCH_ANY), + -EAGAIN, "aligned word at readable page end"); + for (unsigned back = 1; back < sizeof(uint32_t); back++) { + expect(raw_syscall6(__NR_futex, (long) (p + PAGE_SIZE - back), + ops[i] | FUTEX_PRIVATE_FLAG, 1, 0, 0, + FUTEX_BITSET_MATCH_ANY), + -EINVAL, "straddling word is unaligned"); + } + expect(raw_syscall6(__NR_futex, (long) (p + PAGE_SIZE), + ops[i] | FUTEX_PRIVATE_FLAG, 1, 0, 0, + FUTEX_BITSET_MATCH_ANY), + -EFAULT, "aligned word in inaccessible page"); + } + expect(munmap(p, 2 * PAGE_SIZE), 0, "boundary futex munmap"); +} + int main(void) { int word = 1; @@ -177,6 +243,8 @@ int main(void) test_unresolvable_addresses(); test_tagged_address(); + test_lazy_wait_inputs(); + test_page_boundary(); test_matching_word_still_blocks(); if (failures) diff --git a/tests/test-shim-futex-stats.sh b/tests/test-shim-futex-stats.sh index 5a2c427a..6c5711dc 100755 --- a/tests/test-shim-futex-stats.sh +++ b/tests/test-shim-futex-stats.sh @@ -85,9 +85,9 @@ check_positive FUTEX_EAGAIN_HIT check_positive FUTEX_SHAPE_BAIL check_positive FUTEX_MATCH_BAIL -# The EFAULT tail is the riskiest piece of the new assembly: it unwinds a nested -# exception frame by hand. The guest's three unresolvable addresses drive it. -check_positive FUTEX_EFAULT_HIT +# Fault fallback unwinds a nested exception frame before HVC. The guest's +# unresolvable addresses exercise it and must still return EFAULT. +check_positive FUTEX_FAULT_BAIL # The wake path serves the shape a real contended workload actually produces, so # a dispatch that stops reaching it costs more than the wait path does. Its bail diff --git a/tests/test-sysv-shm.c b/tests/test-sysv-shm.c index c657928c..796d66fe 100644 --- a/tests/test-sysv-shm.c +++ b/tests/test-sysv-shm.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -76,11 +77,39 @@ static void test_shm_rdonly_faults_on_write(void) shmctl(shmid, IPC_RMID, NULL); } +static void test_shm_detach_unmapped(void) +{ + TEST("detach after guest munmap"); + const size_t len = 2UL << 20; + int shmid = shmget(IPC_PRIVATE, len, IPC_CREAT | 0600); + if (shmid < 0) { + FAIL("shmget"); + return; + } + void *p = shmat(shmid, NULL, 0); + if (p == (void *) -1) { + FAIL("shmat"); + shmctl(shmid, IPC_RMID, NULL); + return; + } + int unmapped = munmap(p, len); + int detached = shmdt(p); + int detach_errno = errno; + struct shmid_ds info; + /* Linux's munmap already detaches; elfuse retains a host attachment. */ + int ok = unmapped == 0 && + (detached == 0 || (detached == -1 && detach_errno == EINVAL)) && + shmctl(shmid, IPC_STAT, &info) == 0 && info.shm_nattch == 0; + shmctl(shmid, IPC_RMID, NULL); + EXPECT_TRUE(ok, "detach hung, failed, or leaked its host attachment"); +} + int main(void) { printf("test-sysv-shm: SysV shared memory tests\n\n"); test_shm_rdonly_faults_on_write(); + test_shm_detach_unmapped(); SUMMARY("test-sysv-shm"); return fails > 0 ? 1 : 0; diff --git a/tests/test-thread-churn.c b/tests/test-thread-churn.c index f76fc8e4..0d3e511f 100644 --- a/tests/test-thread-churn.c +++ b/tests/test-thread-churn.c @@ -24,9 +24,13 @@ int passes = 0, fails = 0; -#define SEQUENTIAL_ROUNDS 150 +/* Keep this a slot-reuse regression rather than an HVF vCPU lifecycle soak. + * Eighty sequential workers exceed the 64-slot table by 16, and two 32-worker + * batches repeat reuse while unrelated workers remain live. + */ +#define SEQUENTIAL_ROUNDS 80 #define BATCH_SIZE 32 -#define BATCH_ROUNDS 6 +#define BATCH_ROUNDS 2 static void *churn_fn(void *arg) { @@ -40,7 +44,7 @@ static void *churn_fn(void *arg) */ static void test_sequential_churn(void) { - TEST("sequential churn (150 threads)"); + TEST("sequential churn (80 threads)"); for (int i = 0; i < SEQUENTIAL_ROUNDS; i++) { int ran = 0; @@ -67,7 +71,7 @@ static void test_sequential_churn(void) */ static void test_batch_churn(void) { - TEST("batch churn (6x32 threads)"); + TEST("batch churn (2x32 threads)"); for (int round = 0; round < BATCH_ROUNDS; round++) { pthread_t threads[BATCH_SIZE]; diff --git a/tests/test-tlbi-encoder-host.c b/tests/test-tlbi-encoder-host.c index 4757a2f0..d6f55226 100644 --- a/tests/test-tlbi-encoder-host.c +++ b/tests/test-tlbi-encoder-host.c @@ -47,12 +47,12 @@ static void check_field(const char *label, uint64_t got, uint64_t expect) /* Decompose the operand per ARM ARM D8.7.6 and compare each field against the * expected value. baseADDR is VA>>12 masked to 37 bits; TG must be 01 (4 KiB); - * SCALE must be 0; TTL must be 0; ASID must be 0. NUM derives from the page - * count via the ceil(pages/2) - 1 SCALE=0 encoding. + * TTL and ASID must be 0. NUM derives from the selected SCALE unit. */ static void verify_operand(uint64_t start_va, uint16_t pages, - uint64_t expect_num) + uint64_t expect_num, + uint64_t expect_scale) { uint64_t op = tlbi_rvae1is_operand(start_va, pages); @@ -76,7 +76,7 @@ static void verify_operand(uint64_t start_va, check_field(label, num, expect_num); snprintf(label, sizeof(label), "SCALE (pages=%u)", (unsigned) pages); - check_field(label, scale, 0); + check_field(label, scale, expect_scale); snprintf(label, sizeof(label), "TG (start=0x%llx)", (unsigned long long) start_va); @@ -100,30 +100,47 @@ int main(void) * pages 63 -> NUM 31 (covers 64) * pages 64 -> NUM 31 (covers 64) */ - verify_operand(0x10000000ULL, 2, 0); - verify_operand(0x10000000ULL, 3, 1); - verify_operand(0x10000000ULL, 16, 7); - verify_operand(0x10000000ULL, 17, 8); - verify_operand(0x10000000ULL, 32, 15); - verify_operand(0x10000000ULL, 63, 31); - verify_operand(0x10000000ULL, 64, 31); + verify_operand(0x10000000ULL, 2, 0, 0); + verify_operand(0x10000000ULL, 3, 1, 0); + verify_operand(0x10000000ULL, 16, 7, 0); + verify_operand(0x10000000ULL, 17, 8, 0); + verify_operand(0x10000000ULL, 32, 15, 0); + verify_operand(0x10000000ULL, 63, 31, 0); + verify_operand(0x10000000ULL, 64, 31, 0); + + /* SCALE=1 covers 64 pages per NUM step; SCALE=2 covers 2048. A lazy 2 MiB + * block is 512 pages and must therefore encode as SCALE=1, NUM=7. + */ + verify_operand(0x10000000ULL, 512, 7, 1); + verify_operand(0x10000000ULL, 2048, 31, 1); + verify_operand(0x10000000ULL, 8192, 3, 2); /* Boundary VAs. 4 KiB-aligned, low-VA, MMAP_BASE (8 GiB), high-VA just * below the 48-bit BaseADDR truncation point. */ - verify_operand(0x00000000ULL, 32, 15); /* zero base */ - verify_operand(0x200000000ULL, 32, 15); /* MMAP_BASE */ - verify_operand(0x800000000000ULL, 32, 15); /* Rosetta image */ - verify_operand(0x0000FFFFF0000000ULL, 32, 15); /* KBUF_USER_VA */ + verify_operand(0x00000000ULL, 32, 15, 0); /* zero base */ + verify_operand(0x200000000ULL, 32, 15, 0); /* MMAP_BASE */ + verify_operand(0x800000000000ULL, 32, 15, 0); /* Rosetta image */ + verify_operand(0x0000FFFFF0000000ULL, 32, 15, 0); /* KBUF_USER_VA */ /* Pathological inputs the clamp must catch: * pages = 0 -> clamped to 2 -> NUM 0 * pages = 1 -> clamped to 2 -> NUM 0 (callers never reach here) * pages = UINT16_MAX -> NUM clamped to 31 (saturating) */ - verify_operand(0x10000000ULL, 0, 0); - verify_operand(0x10000000ULL, 1, 0); - verify_operand(0x10000000ULL, UINT16_MAX, 31); + verify_operand(0x10000000ULL, 0, 0, 0); + verify_operand(0x10000000ULL, 1, 0, 0); + verify_operand(0x10000000ULL, UINT16_MAX, 31, 2); + + /* The accumulator widens a 2 MiB request to the SCALE=1 granule and keeps + * it on the single-shot RVAE path instead of degrading to broadcast. + */ + g_tlbi_range_supported = true; + tlbi_request_clear(); + tlbi_request_range(0x200000000ULL, 0x200200000ULL); + check_field("2MiB accumulator kind", cpu_tlbi_req.kind, TLBI_RANGE_LARGE); + check_field("2MiB accumulator pages", cpu_tlbi_req.pages, 512); + check_field("2MiB accumulator start", cpu_tlbi_req.start, 0x200000000ULL); /* TG bit is the architectural lynchpin -- if the encoder ever drops it the * integration tests on Apple Silicon would still pass. Pin a direct bit-46 diff --git a/tests/test-usbdev-ioctl.c b/tests/test-usbdev-ioctl.c index 24d8000a..a6ddb3f8 100644 --- a/tests/test-usbdev-ioctl.c +++ b/tests/test-usbdev-ioctl.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -781,6 +782,21 @@ static void check_answers_without_a_device(void) EXPECT_TRUE(io(fd, USBDEVFS_GET_CAPABILITIES, &caps) == 0 && caps == 0, "caps"); + size_t lazy_len = 8UL << 20; + uint8_t *lazy = mmap(NULL, lazy_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + TEST("ioctl and pread into untouched mappings"); + if (lazy == MAP_FAILED) { + FAIL("lazy mmap"); + } else { + uint32_t *lazy_caps = (uint32_t *) (lazy + (2UL << 20)); + EXPECT_TRUE(io(fd, USBDEVFS_GET_CAPABILITIES, lazy_caps) == 0 && + *lazy_caps == 0 && + pread(fd, lazy + (4UL << 20), 18, 0) == 18, + "lazy USB output failed"); + munmap(lazy, lazy_len); + } + TEST("GET_SPEED returns the enum as its value"); EXPECT_EQ(io(fd, USBDEVFS_GET_SPEED, NULL), 2, "get_speed"); /* FULL */