Skip to content

Commit

Permalink
Fix cargo feature 'crashdump'
Browse files Browse the repository at this point in the history
Signed-off-by: Ludvig Liljenberg <lliljenberg@microsoft.com>
  • Loading branch information
ludfjig committed Nov 22, 2024
1 parent a165293 commit e1078a3
Show file tree
Hide file tree
Showing 15 changed files with 164 additions and 229 deletions.
35 changes: 7 additions & 28 deletions .github/workflows/Benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,12 @@ jobs:
strategy:
fail-fast: true
matrix:
include:
- build: windows-2022-release-amd
os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-amd"]
hypervisor: hyperv
arch: amd
- build: linux-kvm-release-amd
os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd"]
hypervisor: kvm
arch: amd
- build: linux-mshv-release-amd
os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-amd"]
hypervisor: mshv
arch: amd
- build: windows-2022-release-intel
os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-intel"]
hypervisor: hyperv
arch: intel
- build: linux-kvm-release-intel
os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-intel"]
hypervisor: kvm
arch: intel
- build: linux-mshv-release-intel
os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-intel"]
hypervisor: mshv
arch: intel
hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux
cpu: [amd, intel]
config: [release] # don't want to benchmark debug-builds

runs-on: ${{ matrix.os }}
runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }}

steps:
### Setup ###

Expand Down Expand Up @@ -82,7 +61,7 @@ jobs:
run: git fetch --tags origin

- name: Download benchmarks from most recent release
run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.arch }} # skip tag parameter to compare to latest stable release
run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu }} # skip tag parameter to compare to latest stable release
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -92,6 +71,6 @@ jobs:

- uses: actions/upload-artifact@v4
with:
name: benchmarks_${{runner.os}}_${{matrix.hypervisor}}_${{ matrix.arch }}
name: benchmarks_${{runner.os}}_${{matrix.hypervisor}}_${{ matrix.cpu }}
path: ./target/criterion/
if-no-files-found: error
8 changes: 8 additions & 0 deletions .github/workflows/ValidatePullRequest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ on:
branches: [main, "release/**"]
merge_group:

# Cancels old running job if a new one is triggered (e.g. by a push onto the same branch).
# This will cancel dependent jobs as well, such as dep_rust and dep_fuzzing
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

permissions:
id-token: write
contents: read
Expand Down Expand Up @@ -42,6 +48,7 @@ jobs:
secrets: inherit
with:
docs_only: ${{needs.docs-pr.outputs.docs-only}}

fuzzing:
needs:
- docs-pr
Expand All @@ -50,6 +57,7 @@ jobs:
max_total_time: 300 # 5 minutes in seconds
docs_only: ${{needs.docs-pr.outputs.docs-only}}
secrets: inherit

spelling:
name: spell check with typos
runs-on: ubuntu-latest
Expand Down
73 changes: 9 additions & 64 deletions .github/workflows/dep_rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,69 +26,11 @@ jobs:
strategy:
fail-fast: true
matrix:
include:
- build: windows-2022-debug-amd
os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-amd"]
hypervisor: hyperv
config: debug
arch: amd
- build: linux-kvm-debug-amd
os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd"]
hypervisor: kvm
config: debug
arch: amd
- build: linux-mshv-debug-amd
os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-amd"]
hypervisor: mshv
config: debug
arch: amd
- build: windows-2022-release-amd
os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-amd"]
hypervisor: hyperv
config: release
arch: amd
- build: linux-kvm-release-amd
os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd"]
hypervisor: kvm
config: release
arch: amd
- build: linux-mshv-release-amd
os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-amd"]
hypervisor: mshv
config: release
arch: amd
- build: windows-2022-debug-intel
os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-intel"]
hypervisor: hyperv
config: debug
arch: intel
- build: linux-kvm-debug-intel
os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-intel"]
hypervisor: kvm
config: debug
arch: intel
- build: linux-mshv-debug-intel
os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-intel"]
hypervisor: mshv
config: debug
arch: intel
- build: windows-2022-release-intel
os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-intel"]
hypervisor: hyperv
config: release
arch: intel
- build: linux-kvm-release-intel
os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-intel"]
hypervisor: kvm
config: release
arch: intel
- build: linux-mshv-release-intel
os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-intel"]
hypervisor: mshv
config: release
arch: intel

runs-on: ${{ matrix.os }}
hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux
cpu: [amd, intel]
config: [debug, release]

runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }}
steps:
- uses: actions/checkout@v4

Expand Down Expand Up @@ -132,6 +74,9 @@ jobs:
# with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess
just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }}
# make sure certain cargo features compile
cargo check -p hyperlight-host --features crashdump
# without any driver (shouldn't compile)
just test-rust-feature-compilation-fail ${{ matrix.config }}
Expand Down Expand Up @@ -160,7 +105,7 @@ jobs:
run: sudo apt install gh -y

- name: Download benchmarks from "latest"
run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.arch}} dev-latest # compare to prerelease
run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu}} dev-latest # compare to prerelease
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
continue-on-error: true
Expand Down
10 changes: 5 additions & 5 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -165,13 +165,13 @@ run-rust-examples-linux target=default-target: (build-rust target) (run-rust-exa
# Downloads the benchmarks result from the given release tag.
# If tag is not given, defaults to latest release
# Options for os: "Windows", or "Linux"
# Options for Linux hypervisor: "kvm", "hyperv"
# Options for Linux hypervisor: "kvm", "mshv"
# Options for Windows hypervisor: "hyperv"
# Options for arch: "amd", "intel"
bench-download os hypervisor arch tag="":
gh release download {{ tag }} -D ./target/ -p benchmarks_{{ os }}_{{ hypervisor }}_{{ arch }}.tar.gz
# Options for cpu: "amd", "intel"
bench-download os hypervisor cpu tag="":
gh release download {{ tag }} -D ./target/ -p benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz
mkdir -p target/criterion {{ if os() == "windows" { "-Force" } else { "" } }}
tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ arch }}.tar.gz -C target/criterion/ --strip-components=1
tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1

# Warning: compares to and then OVERWRITES the given baseline
bench-ci baseline target=default-target:
Expand Down
2 changes: 1 addition & 1 deletion docs/debugging-hyperlight.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de

## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit

To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.

There are no tools at this time to analyze the dump file, but it can be useful for debugging.
4 changes: 1 addition & 3 deletions src/hyperlight_host/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,7 @@ function_call_metrics = []
executable_heap = []
# This feature enables printing of debug information to stdout in debug builds
print_debug = []
# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode
# the name of the file is output to stdout and logged.
dump_on_crash = ["dep:tempfile"]
crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
inprocess = []
Expand Down
2 changes: 2 additions & 0 deletions src/hyperlight_host/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ fn main() -> Result<()> {
// inprocess feature is aliased with debug_assertions to make it only available in debug-builds.
// You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead.
inprocess: { all(feature = "inprocess", debug_assertions) },
// crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
crashdump: { all(feature = "crashdump", debug_assertions) },
}

write_built_file()?;
Expand Down
47 changes: 47 additions & 0 deletions src/hyperlight_host/src/hypervisor/crashdump.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
use std::io::Write;

use tempfile::NamedTempFile;

use super::Hypervisor;
use crate::{new_error, Result};

/// Dump registers + memory regions + raw memory to a tempfile
#[cfg(crashdump)]
pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
let mem_regions = hv.get_memory_regions();
let mem_size = mem_regions
.iter()
.map(|region| region.host_region.len())
.sum();
let mem_start_addr = mem_regions[0].host_region.start as *const u8;

if mem_start_addr.is_null() || mem_size == 0 {
return Err(new_error!(
"Invalid address or size while creating crashdump"
));
}

let mut temp_file = NamedTempFile::with_prefix("mem")?;

let hv_details = format!("{:#x?}", hv);

// write hypervisor details such as registers, memory regions, etc.
temp_file.write_all(hv_details.as_bytes())?;
// write memory dump
temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
// SAFETY: Address and size non-null and non-zero
unsafe {
let slice = std::slice::from_raw_parts(mem_start_addr, mem_size);
temp_file.write_all(slice)?;
temp_file.flush()?;
}
let persist_path = temp_file.path().with_extension("dmp");
temp_file
.persist(&persist_path)
.map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;

println!("Memory dumped to file: {:?}", persist_path);
log::error!("Memory dumped to file: {:?}", persist_path);

Ok(())
}
13 changes: 5 additions & 8 deletions src/hyperlight_host/src/hypervisor/hyperv_linux.rs
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,6 @@ impl Hypervisor for HypervLinuxDriver {
"mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}",
addr, &self
);
#[cfg(all(debug_assertions, feature = "dump_on_crash"))]
self.dump_on_crash(self.mem_regions.clone());
HyperlightExit::Mmio(addr)
}
INVALID_GPA_ACCESS_MESSAGE => {
Expand All @@ -323,8 +321,6 @@ impl Hypervisor for HypervLinuxDriver {
"mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}",
gpa, &self
);
#[cfg(all(debug_assertions, feature = "dump_on_crash"))]
self.dump_on_crash(self.mem_regions.clone());
match self.get_memory_access_violation(
gpa as usize,
&self.mem_regions,
Expand All @@ -336,8 +332,6 @@ impl Hypervisor for HypervLinuxDriver {
}
other => {
debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
#[cfg(all(debug_assertions, feature = "dump_on_crash"))]
self.dump_on_crash(self.mem_regions.clone());
log_then_return!("unknown Hyper-V run message type {:?}", other);
}
},
Expand All @@ -347,8 +341,6 @@ impl Hypervisor for HypervLinuxDriver {
libc::EAGAIN => HyperlightExit::Retry(),
_ => {
debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
#[cfg(all(debug_assertions, feature = "dump_on_crash"))]
self.dump_on_crash(self.mem_regions.clone());
log_then_return!("Error running VCPU {:?}", e);
}
},
Expand All @@ -360,6 +352,11 @@ impl Hypervisor for HypervLinuxDriver {
fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
self as &mut dyn Hypervisor
}

#[cfg(crashdump)]
fn get_memory_regions(&self) -> &[MemoryRegion] {
&self.mem_regions
}
}

impl Drop for HypervLinuxDriver {
Expand Down
29 changes: 5 additions & 24 deletions src/hyperlight_host/src/hypervisor/hyperv_windows.rs
Original file line number Diff line number Diff line change
Expand Up @@ -508,18 +508,6 @@ impl Hypervisor for HypervWindowsDriver {
"HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ",
gpa, access_info, &self
);
#[cfg(all(debug_assertions, feature = "dump_on_crash"))]
{
if let Err(e) = unsafe {
self.write_dump_file(
self.mem_regions.clone(),
self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
self.size,
)
} {
println!("Error dumping memory: {}", e);
}
}

match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info)
{
Expand All @@ -539,18 +527,6 @@ impl Hypervisor for HypervWindowsDriver {
"HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}",
exit_context.ExitReason, &self
);
#[cfg(all(debug_assertions, feature = "dump_on_crash"))]
{
if let Err(e) = unsafe {
self.write_dump_file(
self.mem_regions.clone(),
self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
self.size,
)
} {
println!("Error dumping memory: {}", e);
}
}
match self.get_exit_details(exit_context.ExitReason) {
Ok(error) => HyperlightExit::Unknown(error),
Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)),
Expand All @@ -569,6 +545,11 @@ impl Hypervisor for HypervWindowsDriver {
fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
self as &mut dyn Hypervisor
}

#[cfg(crashdump)]
fn get_memory_regions(&self) -> &[MemoryRegion] {
&self.mem_regions
}
}

#[cfg(test)]
Expand Down
7 changes: 7 additions & 0 deletions src/hyperlight_host/src/hypervisor/inprocess.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ use std::fmt::Debug;
use std::os::raw::c_void;

use super::{HyperlightExit, Hypervisor};
#[cfg(crashdump)]
use crate::mem::memory_region::MemoryRegion;
use crate::sandbox::leaked_outb::LeakedOutBWrapper;
use crate::Result;

Expand Down Expand Up @@ -123,4 +125,9 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE {
unimplemented!("get_partition_handle should not be needed since we are in in-process mode")
}

#[cfg(crashdump)]
fn get_memory_regions(&self) -> &[MemoryRegion] {
unimplemented!("get_memory_regions is not supported since we are in in-process mode")
}
}
Loading

0 comments on commit e1078a3

Please sign in to comment.