diff --git a/.vscode/settings.json b/.vscode/settings.json index 22ee1897..5d2a7ff1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,15 +8,15 @@ "clippy", "--quiet", "--message-format=json", - "--target=aarch64-unknown-none-softfloat", - "--target=riscv64imac-unknown-none-elf", + // "--target=aarch64-unknown-none-softfloat", + // "--target=riscv64imac-unknown-none-elf", "--target=x86_64-unknown-none", - "--target=x86_64-unknown-uefi", + // "--target=x86_64-unknown-uefi", ], "rust-analyzer.check.targets": [ - "aarch64-unknown-none-softfloat", - "riscv64imac-unknown-none-elf", + // "aarch64-unknown-none-softfloat", + // "riscv64imac-unknown-none-elf", "x86_64-unknown-none", - "x86_64-unknown-uefi", + // "x86_64-unknown-uefi", ] } \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 06d87166..0241afb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ built = { version = "0.8", features = ["git2", "chrono"] } # FIXME: When target-specific features exist, remove the workaround features. # https://github.com/rust-lang/cargo/issues/1197 [features] -default = [] +default = ["x86_64-linux"] elf = [] linux = [] multiboot = [] diff --git a/data/x86_64/hello_world b/data/x86_64/hello_world index 823b11c7..6b209000 100755 --- a/data/x86_64/hello_world +++ b/data/x86_64/hello_world @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eeb3809d55f20293260ced7efefa3cb5debea8b465990b505a6252ee729766c6 -size 921816 +oid sha256:be3c1014088e5e56cd30b16126633e0596d1686cb4ad55aa4212cbaaadd69fe7 +size 924128 diff --git a/data/x86_64/hello_world-microvm b/data/x86_64/hello_world-microvm index 7147ae3b..bbf28be8 100755 --- a/data/x86_64/hello_world-microvm +++ b/data/x86_64/hello_world-microvm @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f79a47ec86044cacf471e65875d48a66017cf269bd42a33b4e21def2b3b229d6 -size 920976 +oid sha256:6cd50fd386ece311aa8db3e9adebe762347ebc979762f150c19d6764694887dc +size 875544 diff --git a/src/arch/x86_64/page_tables.rs b/src/arch/x86_64/page_tables.rs index b8991040..b215801a 100644 --- a/src/arch/x86_64/page_tables.rs +++ b/src/arch/x86_64/page_tables.rs @@ -23,9 +23,17 @@ //! //! [rust-lang/rust#51910 (comment)]: https://github.com/rust-lang/rust/issues/51910#issuecomment-1013271838 -use core::ptr; +use core::ops::Range; +use core::{fmt, ptr}; -use x86_64::structures::paging::{PageSize, PageTableFlags, Size2MiB}; +use log::{debug, info, warn}; +use x86_64::structures::paging::{ + Mapper, OffsetPageTable, PageSize, PageTableFlags, PhysFrame, Size1GiB, Size2MiB, +}; +use x86_64::{PhysAddr, VirtAddr}; + +use self::cpuid::ExtendedProcessorAndProcessorFeatureIdentifiers; +use crate::arch::x86_64::physicalmem::PhysAlloc; const TABLE_FLAGS: PageTableFlags = PageTableFlags::PRESENT.union(PageTableFlags::WRITABLE); const PAGE_FLAGS: PageTableFlags = TABLE_FLAGS.union(PageTableFlags::HUGE_PAGE); @@ -65,6 +73,109 @@ static mut LEVEL_2_TABLE: PageTable = { PageTable(page_table) }; +/// Initializes the page tables. +/// +/// # Safety +/// +/// This function may only be called once before modifying the page tables. +pub unsafe fn init(max_phys_addr: usize) { + debug!("max_phys_addr = {max_phys_addr:#x}"); + + let idents = ExtendedProcessorAndProcessorFeatureIdentifiers::new(); + let has_page_1_gb = idents.has_page_1_gb(); + + if has_page_1_gb { + info!("CPU supports 1-GiB pages."); + } else { + warn!("CPU does not support 1-GiB pages."); + } + + if has_page_1_gb { + // If supported, we replace the existing mapping of 512 2-MiB pages with 1 1-GiB page. + // + // Since the mappings themselves do not change, we don't need to flush the TLB. + // For details, see Section 5.10.2.3 "Details of TLB Use" in the Intel® 64 and IA-32 + // Architectures Software Developer's Manual Volume 3A: System Programming Guide, Part 1. + + info!("Replacing the 2-MiB pages with a 1-GiB page."); + + let flags: usize = PAGE_FLAGS.bits() as usize; + let addr = 0; + unsafe { + LEVEL_3_TABLE.0[0] = ptr::with_exposed_provenance_mut(addr + flags); + } + } + + let addrs = Size1GiB::SIZE as usize..max_phys_addr; + + if has_page_1_gb { + identity_map::(addrs); + } else { + identity_map::(addrs); + } +} + +fn identity_map(phys_addrs: Range) +where + for<'a> OffsetPageTable<'a>: Mapper, +{ + if phys_addrs.end <= phys_addrs.start { + return; + } + + let start_addr = PhysAddr::new(phys_addrs.start as u64); + let last_addr = PhysAddr::new((phys_addrs.end - 1) as u64); + + let start = PhysFrame::::from_start_address(start_addr).unwrap(); + let last = PhysFrame::::containing_address(last_addr); + + info!("Identity-mapping {start:?}..={last:?}"); + + let frames = PhysFrame::range_inclusive(start, last); + + let level_4_table = unsafe { &mut *(&raw mut LEVEL_4_TABLE).cast() }; + let phys_offset = VirtAddr::new(0); + let mut page_table = unsafe { OffsetPageTable::new(level_4_table, phys_offset) }; + + let flags = PageTableFlags::PRESENT | PageTableFlags::WRITABLE; + + for frame in frames { + // SAFETY: We are mapping unused pages to unused frames. + let result = unsafe { page_table.identity_map(frame, flags, &mut PhysAlloc) }; + + // This page was not mapped previously. + // Thus, we don't need to flush the TLB. + result.unwrap().ignore(); + } +} + #[repr(align(0x1000))] #[repr(C)] pub struct PageTable([*mut (); 512]); + +mod cpuid { + use core::arch::x86_64::CpuidResult; + + /// Extended Processor and Processor Feature Identifiers + /// + /// We could also use the `raw-cpuid` crate instead, but it is slower, bigger, and less ergonomic. + #[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] + pub struct ExtendedProcessorAndProcessorFeatureIdentifiers(CpuidResult); + + impl ExtendedProcessorAndProcessorFeatureIdentifiers { + const FUNCTION: u32 = 0x8000_0001; + + pub fn new() -> Self { + let cpuid_result = unsafe { core::arch::x86_64::__cpuid(Self::FUNCTION) }; + Self(cpuid_result) + } + + /// 1-GB large page support. + #[doc(alias = "Page1GB")] + pub fn has_page_1_gb(&self) -> bool { + const PAGE_1_GB: u32 = 1 << 26; + + self.0.edx & PAGE_1_GB == PAGE_1_GB + } + } +} diff --git a/src/arch/x86_64/platform/linux/mod.rs b/src/arch/x86_64/platform/linux/mod.rs index 72318781..c48d14df 100644 --- a/src/arch/x86_64/platform/linux/mod.rs +++ b/src/arch/x86_64/platform/linux/mod.rs @@ -15,7 +15,7 @@ use x86_64::structures::paging::{PageSize, PageTableFlags, Size2MiB, Size4KiB}; use crate::BootInfoExt; use crate::arch::x86_64::physicalmem::PhysAlloc; -use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, paging}; +use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, page_tables, paging}; use crate::fdt::Fdt; unsafe extern "C" { @@ -40,6 +40,26 @@ static BOOT_PARAMS: AtomicPtr = AtomicPtr::new(ptr::null_mut()); unsafe extern "C" fn rust_start(boot_params: *mut BootParams) -> ! { crate::log::init(); BOOT_PARAMS.store(boot_params, Ordering::Relaxed); + + let free_addr = ptr::addr_of!(loader_end) + .addr() + .align_up(Size2MiB::SIZE as usize); + // Memory after the highest end address is unused and available for the physical memory manager. + info!("Intializing PhysAlloc with {free_addr:#x}"); + PhysAlloc::init(free_addr); + + let boot_params_ref = unsafe { BootParams::get() }; + let e820_entries = boot_params_ref.e820_entries(); + let max_phys_addr = e820_entries + .iter() + .copied() + .map(|entry| entry.addr + entry.size) + .max() + .unwrap(); + unsafe { + page_tables::init(max_phys_addr.try_into().unwrap()); + } + unsafe { crate::os::loader_main(); } @@ -55,13 +75,6 @@ pub fn find_kernel() -> &'static [u8] { assert!(boot_params_ref.supported()); - let free_addr = ptr::addr_of!(loader_end) - .addr() - .align_up(Size2MiB::SIZE as usize); - // Memory after the highest end address is unused and available for the physical memory manager. - info!("Intializing PhysAlloc with {free_addr:#x}"); - PhysAlloc::init(free_addr); - boot_params_ref.map_ramdisk().unwrap() } diff --git a/src/arch/x86_64/platform/multiboot/mod.rs b/src/arch/x86_64/platform/multiboot/mod.rs index 59dc542e..467b632e 100644 --- a/src/arch/x86_64/platform/multiboot/mod.rs +++ b/src/arch/x86_64/platform/multiboot/mod.rs @@ -9,13 +9,13 @@ use hermit_entry::boot_info::{ }; use hermit_entry::elf::LoadedKernel; use log::info; -use multiboot::information::{MemoryManagement, Multiboot, MultibootInfo, PAddr}; +use multiboot::information::{MemoryManagement, MemoryType, Multiboot, MultibootInfo, PAddr}; use vm_fdt::FdtWriterResult; use x86_64::structures::paging::{PageSize, PageTableFlags, Size2MiB, Size4KiB}; use crate::BootInfoExt; use crate::arch::x86_64::physicalmem::PhysAlloc; -use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, paging}; +use crate::arch::x86_64::{KERNEL_STACK_SIZE, SERIAL_IO_PORT, page_tables, paging}; use crate::fdt::Fdt; unsafe extern "C" { @@ -41,6 +41,25 @@ static MB_INFO: AtomicPtr = AtomicPtr::new(ptr::null_mut()); unsafe extern "C" fn rust_start(mb_info: *mut MultibootInfo) -> ! { crate::log::init(); MB_INFO.store(mb_info, Ordering::Relaxed); + + let mut mem = Mem; + let multiboot = unsafe { Multiboot::from_ref(&mut *mb_info, &mut mem) }; + let highest_address = multiboot.find_highest_address().align_up(Size2MiB::SIZE) as usize; + // Memory after the highest end address is unused and available for the physical memory manager. + PhysAlloc::init(highest_address); + + let max_phys_addr = multiboot + .memory_regions() + .unwrap() + .filter(|memory_region| memory_region.memory_type() == MemoryType::Available) + .map(|memory_region| memory_region.base_address() + memory_region.length()) + .max() + .unwrap(); + + unsafe { + page_tables::init(max_phys_addr.try_into().unwrap()); + } + unsafe { crate::os::loader_main(); } @@ -91,8 +110,6 @@ impl DeviceTree { } pub fn find_kernel() -> &'static [u8] { - use core::cmp; - paging::clean_up(); // Identity-map the Multiboot information. let mb_info = MB_INFO.load(Ordering::Relaxed); @@ -126,15 +143,7 @@ pub fn find_kernel() -> &'static [u8] { let elf_len = (first_module.end - first_module.start) as usize; info!("Module length: {elf_len:#x}"); - // Find the maximum end address from the remaining modules - let mut end_address = first_module.end; - for m in module_iter { - end_address = cmp::max(end_address, m.end); - } - - let modules_mapping_end = end_address.align_up(Size2MiB::SIZE) as usize; - // Memory after the highest end address is unused and available for the physical memory manager. - PhysAlloc::init(modules_mapping_end); + let highest_address = multiboot.find_highest_address().align_up(Size2MiB::SIZE) as usize; // Identity-map the ELF header of the first module and until the 2 MiB // mapping starts. We cannot start the 2 MiB mapping right from @@ -152,7 +161,7 @@ pub fn find_kernel() -> &'static [u8] { paging::map_range::( first_module_mapping_end, first_module_mapping_end, - modules_mapping_end, + highest_address, PageTableFlags::empty(), ); diff --git a/xtask/src/ci/firecracker_vm_config.json b/xtask/src/ci/firecracker_vm_config.json index 14c7b8e2..2391e819 100644 --- a/xtask/src/ci/firecracker_vm_config.json +++ b/xtask/src/ci/firecracker_vm_config.json @@ -7,7 +7,7 @@ "drives": [], "machine-config": {{ "vcpu_count": 1, - "mem_size_mib": 256, + "mem_size_mib": 8196, "smt": false }} }} diff --git a/xtask/src/ci/qemu.rs b/xtask/src/ci/qemu.rs index e18b3e41..6985efd9 100644 --- a/xtask/src/ci/qemu.rs +++ b/xtask/src/ci/qemu.rs @@ -288,7 +288,7 @@ impl Qemu { } _ => {} } - memory + 8196 } fn memory_args(&self) -> [String; 2] {