feat(x86_64): identity-map the whole RAM

.vscode/settings.json

@@ -8,15 +8,15 @@
         "clippy",
         "--quiet",
         "--message-format=json",
-        "--target=aarch64-unknown-none-softfloat",
-        "--target=riscv64imac-unknown-none-elf",
+        // "--target=aarch64-unknown-none-softfloat",
+        // "--target=riscv64imac-unknown-none-elf",
         "--target=x86_64-unknown-none",
-        "--target=x86_64-unknown-uefi",
+        // "--target=x86_64-unknown-uefi",
     ],
     "rust-analyzer.check.targets": [
-        "aarch64-unknown-none-softfloat",
-        "riscv64imac-unknown-none-elf",
+        // "aarch64-unknown-none-softfloat",
+        // "riscv64imac-unknown-none-elf",
         "x86_64-unknown-none",
-        "x86_64-unknown-uefi",
+        // "x86_64-unknown-uefi",
     ]
 }

Cargo.toml

@@ -46,7 +46,7 @@ built = { version = "0.8", features = ["git2", "chrono"] }
 # FIXME: When target-specific features exist, remove the workaround features.
 # https://github.com/rust-lang/cargo/issues/1197
 [features]
-default = []
+default = ["x86_64-linux"]
 elf = []
 linux = []
 multiboot = []

src/arch/x86_64/page_tables.rs

@@ -23,9 +23,17 @@
 //!
 //! [rust-lang/rust#51910 (comment)]: https://github.com/rust-lang/rust/issues/51910#issuecomment-1013271838
 
-use core::ptr;
+use core::ops::Range;
+use core::{fmt, ptr};
 
-use x86_64::structures::paging::{PageSize, PageTableFlags, Size2MiB};
+use log::{debug, info, warn};
+use x86_64::structures::paging::{
+	Mapper, OffsetPageTable, PageSize, PageTableFlags, PhysFrame, Size1GiB, Size2MiB,
+};
+use x86_64::{PhysAddr, VirtAddr};
+
+use self::cpuid::ExtendedProcessorAndProcessorFeatureIdentifiers;
+use crate::arch::x86_64::physicalmem::PhysAlloc;
 
 const TABLE_FLAGS: PageTableFlags = PageTableFlags::PRESENT.union(PageTableFlags::WRITABLE);
 const PAGE_FLAGS: PageTableFlags = TABLE_FLAGS.union(PageTableFlags::HUGE_PAGE);
@@ -65,6 +73,109 @@ static mut LEVEL_2_TABLE: PageTable = {
 	PageTable(page_table)
 };
 
+/// Initializes the page tables.
+///
+/// # Safety
+///
+/// This function may only be called once before modifying the page tables.
+pub unsafe fn init(max_phys_addr: usize) {
+	debug!("max_phys_addr = {max_phys_addr:#x}");
+
+	let idents = ExtendedProcessorAndProcessorFeatureIdentifiers::new();
+	let has_page_1_gb = idents.has_page_1_gb();
+
+	if has_page_1_gb {
+		info!("CPU supports 1-GiB pages.");
+	} else {
+		warn!("CPU does not support 1-GiB pages.");
+	}
+
+	if has_page_1_gb {
+		// If supported, we replace the existing mapping of 512 2-MiB pages with 1 1-GiB page.
+		//
+		// Since the mappings themselves do not change, we don't need to flush the TLB.
+		// For details, see Section 5.10.2.3 "Details of TLB Use" in the Intel® 64 and IA-32
+		// Architectures Software Developer's Manual Volume 3A: System Programming Guide, Part 1.
+
+		info!("Replacing the 2-MiB pages with a 1-GiB page.");
+
+		let flags: usize = PAGE_FLAGS.bits() as usize;
+		let addr = 0;
+		unsafe {
+			LEVEL_3_TABLE.0[0] = ptr::with_exposed_provenance_mut(addr + flags);
+		}
+	}
+
+	let addrs = Size1GiB::SIZE as usize..max_phys_addr;
+
+	if has_page_1_gb {
+		identity_map::<Size1GiB>(addrs);
+	} else {
+		identity_map::<Size2MiB>(addrs);
+	}
+}
+
+fn identity_map<S: PageSize + fmt::Debug>(phys_addrs: Range<usize>)
+where
+	for<'a> OffsetPageTable<'a>: Mapper<S>,
+{
+	if phys_addrs.end <= phys_addrs.start {
+		return;
+	}
+
+	let start_addr = PhysAddr::new(phys_addrs.start as u64);
+	let last_addr = PhysAddr::new((phys_addrs.end - 1) as u64);
+
+	let start = PhysFrame::<S>::from_start_address(start_addr).unwrap();
+	let last = PhysFrame::<S>::containing_address(last_addr);
+
+	info!("Identity-mapping {start:?}..={last:?}");
+
+	let frames = PhysFrame::range_inclusive(start, last);
+
+	let level_4_table = unsafe { &mut *(&raw mut LEVEL_4_TABLE).cast() };
+	let phys_offset = VirtAddr::new(0);
+	let mut page_table = unsafe { OffsetPageTable::new(level_4_table, phys_offset) };
+
+	let flags = PageTableFlags::PRESENT | PageTableFlags::WRITABLE;
+
+	for frame in frames {
+		// SAFETY: We are mapping unused pages to unused frames.
+		let result = unsafe { page_table.identity_map(frame, flags, &mut PhysAlloc) };
+
+		// This page was not mapped previously.
+		// Thus, we don't need to flush the TLB.
+		result.unwrap().ignore();
+	}
+}
+
 #[repr(align(0x1000))]
 #[repr(C)]
 pub struct PageTable([*mut (); 512]);
+
+mod cpuid {
+	use core::arch::x86_64::CpuidResult;
+
+	/// Extended Processor and Processor Feature Identifiers
+	///
+	/// We could also use the `raw-cpuid` crate instead, but it is slower, bigger, and less ergonomic.
+	#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+	pub struct ExtendedProcessorAndProcessorFeatureIdentifiers(CpuidResult);
+
+	impl ExtendedProcessorAndProcessorFeatureIdentifiers {
+		const FUNCTION: u32 = 0x8000_0001;
+
+		pub fn new() -> Self {
+			let cpuid_result = unsafe { core::arch::x86_64::__cpuid(Self::FUNCTION) };
+			Self(cpuid_result)
+		}
+
+		/// 1-GB large page support.
+		#[doc(alias = "Page1GB")]
+		pub fn has_page_1_gb(&self) -> bool {
+			const PAGE_1_GB: u32 = 1 << 26;
+
+			self.0.edx & PAGE_1_GB == PAGE_1_GB
+		}
+	}
+}

src/arch/x86_64/platform/linux/mod.rs

@@ -15,7 +15,7 @@ use x86_64::structures::paging::{PageSize, PageTableFlags, Size2MiB, Size4KiB};
 
 use crate::BootInfoExt;
 use crate::arch::x86_64::physicalmem::PhysAlloc;
-use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, paging};
+use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, page_tables, paging};
 use crate::fdt::Fdt;
 
 unsafe extern "C" {
@@ -40,6 +40,26 @@ static BOOT_PARAMS: AtomicPtr<BootParams> = AtomicPtr::new(ptr::null_mut());
 unsafe extern "C" fn rust_start(boot_params: *mut BootParams) -> ! {
 	crate::log::init();
 	BOOT_PARAMS.store(boot_params, Ordering::Relaxed);
+
+	let free_addr = ptr::addr_of!(loader_end)
+		.addr()
+		.align_up(Size2MiB::SIZE as usize);
+	// Memory after the highest end address is unused and available for the physical memory manager.
+	info!("Intializing PhysAlloc with {free_addr:#x}");
+	PhysAlloc::init(free_addr);
+
+	let boot_params_ref = unsafe { BootParams::get() };
+	let e820_entries = boot_params_ref.e820_entries();
+	let max_phys_addr = e820_entries
+		.iter()
+		.copied()
+		.map(|entry| entry.addr + entry.size)
+		.max()
+		.unwrap();
+	unsafe {
+		page_tables::init(max_phys_addr.try_into().unwrap());
+	}
+
 	unsafe {
 		crate::os::loader_main();
 	}
@@ -55,13 +75,6 @@ pub fn find_kernel() -> &'static [u8] {
 
 	assert!(boot_params_ref.supported());
 
-	let free_addr = ptr::addr_of!(loader_end)
-		.addr()
-		.align_up(Size2MiB::SIZE as usize);
-	// Memory after the highest end address is unused and available for the physical memory manager.
-	info!("Intializing PhysAlloc with {free_addr:#x}");
-	PhysAlloc::init(free_addr);
-
 	boot_params_ref.map_ramdisk().unwrap()
 }
 

src/arch/x86_64/platform/multiboot/mod.rs

@@ -9,13 +9,13 @@ use hermit_entry::boot_info::{
 };
 use hermit_entry::elf::LoadedKernel;
 use log::info;
-use multiboot::information::{MemoryManagement, Multiboot, MultibootInfo, PAddr};
+use multiboot::information::{MemoryManagement, MemoryType, Multiboot, MultibootInfo, PAddr};
 use vm_fdt::FdtWriterResult;
 use x86_64::structures::paging::{PageSize, PageTableFlags, Size2MiB, Size4KiB};
 
 use crate::BootInfoExt;
 use crate::arch::x86_64::physicalmem::PhysAlloc;
-use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, paging};
+use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, page_tables, paging};
 use crate::fdt::Fdt;
 
 unsafe extern "C" {
@@ -41,6 +41,25 @@ static MB_INFO: AtomicPtr<MultibootInfo> = AtomicPtr::new(ptr::null_mut());
 unsafe extern "C" fn rust_start(mb_info: *mut MultibootInfo) -> ! {
 	crate::log::init();
 	MB_INFO.store(mb_info, Ordering::Relaxed);
+
+	let mut mem = Mem;
+	let multiboot = unsafe { Multiboot::from_ref(&mut *mb_info, &mut mem) };
+	let highest_address = multiboot.find_highest_address().align_up(Size2MiB::SIZE) as usize;
+	// Memory after the highest end address is unused and available for the physical memory manager.
+	PhysAlloc::init(highest_address);
+
+	let max_phys_addr = multiboot
+		.memory_regions()
+		.unwrap()
+		.filter(|memory_region| memory_region.memory_type() == MemoryType::Available)
+		.map(|memory_region| memory_region.base_address() + memory_region.length())
+		.max()
+		.unwrap();
+
+	unsafe {
+		page_tables::init(max_phys_addr.try_into().unwrap());
+	}
+
 	unsafe {
 		crate::os::loader_main();
 	}
@@ -91,8 +110,6 @@ impl DeviceTree {
 }
 
 pub fn find_kernel() -> &'static [u8] {
-	use core::cmp;
-
 	paging::clean_up();
 	// Identity-map the Multiboot information.
 	let mb_info = MB_INFO.load(Ordering::Relaxed);
@@ -126,15 +143,7 @@ pub fn find_kernel() -> &'static [u8] {
 	let elf_len = (first_module.end - first_module.start) as usize;
 	info!("Module length: {elf_len:#x}");
 
-	// Find the maximum end address from the remaining modules
-	let mut end_address = first_module.end;
-	for m in module_iter {
-		end_address = cmp::max(end_address, m.end);
-	}
-
-	let modules_mapping_end = end_address.align_up(Size2MiB::SIZE) as usize;
-	// Memory after the highest end address is unused and available for the physical memory manager.
-	PhysAlloc::init(modules_mapping_end);
+	let highest_address = multiboot.find_highest_address().align_up(Size2MiB::SIZE) as usize;
 
 	// Identity-map the ELF header of the first module and until the 2 MiB
 	// mapping starts. We cannot start the 2 MiB mapping right from
@@ -152,7 +161,7 @@ pub fn find_kernel() -> &'static [u8] {
 	paging::map_range::<Size2MiB>(
 		first_module_mapping_end,
 		first_module_mapping_end,
-		modules_mapping_end,
+		highest_address,
 		PageTableFlags::empty(),
 	);
 

xtask/src/ci/firecracker_vm_config.json

@@ -7,7 +7,7 @@
   "drives": [],
   "machine-config": {{
     "vcpu_count": 1,
-    "mem_size_mib": 256,
+    "mem_size_mib": 8196,
     "smt": false
   }}
 }}

xtask/src/ci/qemu.rs

@@ -288,7 +288,7 @@ impl Qemu {
 			}
 			_ => {}
 		}
-		memory
+		8196
 	}
 
 	fn memory_args(&self) -> [String; 2] {