Publish instance vCPU stats to oximeter (#659)

* Publish instance vCPU stats to oximeter

- Add more metadata to instance and HTTP API, for tracking silo /
  project IDs for an instance.
- Add the `VirtualMachine` kstat target for tracking individual vCPU
  usage by microstate.
- Use `VirtualMachine` in place of the bare instance UUID for tracking
  other statistics too.
- Use `KstatSampler` to pull vCPU kernel statistics to track vCPU usage
- A few minor tweaks to existing metrics
- Update OpenAPI documents

* Will it blend on linux?

* Review feedback

- Merge various use statements
- Docstring improvements
- Ref issue tracking PVPANIC statistic organization
- Add oximeter state to `ServiceProviders`, storing registration task
  handle. Abort the registration task in `stop()` method, like the
  others.

* Review feedback

- Move lock around oximeter state up one layer
- Add mechanism for the `ServiceProviders::stop()` method to notify the
  metric registration task to bail if needed.

* Correct const for number of tracked vCPU microstates

* More review feedback

- Added test to catch changes in kstate->oximeter vCPU microstate
  mapping
- Make oximeter state names consts
- Define the number of expected states explicitly as the len of an array
  of the state names.

* Use typed UUID wrapper

Author: bnaecker

Cargo.lock

@@ -73,6 +73,7 @@ internal-dns = { git = "https://github.com/oxidecomputer/omicron", branch = "mai
 nexus-client = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 omicron-common = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 omicron-zone-package = "0.9.0"
+oximeter-instruments = { git = "https://github.com/oxidecomputer/omicron", branch = "main", default-features = false, features = ["kstat"] }
 oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 
@@ -115,9 +116,11 @@ http = "0.2.9"
 hyper = "0.14"
 indicatif = "0.17.3"
 inventory = "0.3.0"
+kstat-rs = "0.2.3"
 lazy_static = "1.4"
 libc = "0.2"
 mockall = "0.11"
+newtype-uuid = { version = "1.0.1", features = [ "v4" ] }
 num_enum = "0.5.11"
 owo-colors = "4"
 pin-project-lite = "0.2.13"

Cargo.toml

@@ -9,6 +9,16 @@ progenitor::generate_api!(
     spec = "../../openapi/propolis-server.json",
     derives = [schemars::JsonSchema],
     patch = {
+        InstanceMetadata = {
+            derives = [
+                Clone,
+                schemars::JsonSchema,
+                Serialize,
+                Deserialize,
+                Eq,
+                PartialEq,
+            ]
+        },
         InstanceProperties = {
             derives = [
                 Clone,

bin/mock-server/src/lib/api_types.rs

@@ -9,6 +9,7 @@ anyhow.workspace = true
 clap = { workspace = true, features = ["derive"] }
 futures.workspace = true
 libc.workspace = true
+newtype-uuid.workspace = true
 propolis-client.workspace = true
 slog.workspace = true
 slog-async.workspace = true

bin/propolis-cli/Cargo.toml

@@ -14,6 +14,8 @@ use std::{
 use anyhow::{anyhow, Context};
 use clap::{Parser, Subcommand};
 use futures::{future, SinkExt};
+use newtype_uuid::{GenericUuid, TypedUuid, TypedUuidKind, TypedUuidTag};
+use propolis_client::types::InstanceMetadata;
 use slog::{o, Drain, Level, Logger};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio_tungstenite::tungstenite::{
@@ -79,6 +81,14 @@ enum Command {
         // cloud_init ISO file
         #[clap(long, action)]
         cloud_init: Option<PathBuf>,
+
+        /// A UUID to use for the instance's silo, attached to instance metrics.
+        #[clap(long)]
+        silo_id: Option<TypedUuid<SiloKind>>,
+
+        /// A UUID to use for the instance's project, attached to instance metrics.
+        #[clap(long)]
+        project_id: Option<TypedUuid<ProjectKind>>,
     },
 
     /// Get the properties of a propolis instance
@@ -178,6 +188,26 @@ fn create_logger(opt: &Opt) -> Logger {
     Logger::root(drain, o!())
 }
 
+// Implement typed UUID wrappers for the project / silo IDs, to avoid conflating
+// them.
+enum ProjectKind {}
+
+impl TypedUuidKind for ProjectKind {
+    fn tag() -> TypedUuidTag {
+        const TAG: TypedUuidTag = TypedUuidTag::new("project");
+        TAG
+    }
+}
+
+enum SiloKind {}
+
+impl TypedUuidKind for SiloKind {
+    fn tag() -> TypedUuidTag {
+        const TAG: TypedUuidTag = TypedUuidTag::new("silo");
+        TAG
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 async fn new_instance(
     client: &Client,
@@ -187,11 +217,17 @@ async fn new_instance(
     memory: u64,
     disks: Vec<DiskRequest>,
     cloud_init_bytes: Option<String>,
+    silo_id: TypedUuid<SiloKind>,
+    project_id: TypedUuid<ProjectKind>,
 ) -> anyhow::Result<()> {
     let properties = InstanceProperties {
         id,
         name,
         description: "propolis-cli generated instance".to_string(),
+        metadata: InstanceMetadata {
+            silo_id: silo_id.into_untyped_uuid(),
+            project_id: project_id.into_untyped_uuid(),
+        },
         // TODO: Use real UUID
         image_id: Uuid::default(),
         // TODO: Use real UUID
@@ -620,6 +656,8 @@ async fn main() -> anyhow::Result<()> {
             memory,
             crucible_disks,
             cloud_init,
+            silo_id,
+            project_id,
         } => {
             let disks = if let Some(crucible_disks) = crucible_disks {
                 parse_json_file(&crucible_disks)?
@@ -642,6 +680,8 @@ async fn main() -> anyhow::Result<()> {
                 memory,
                 disks,
                 cloud_init_bytes,
+                silo_id.unwrap_or_else(TypedUuid::new_v4),
+                project_id.unwrap_or_else(TypedUuid::new_v4),
             )
             .await?
         }

bin/propolis-cli/src/main.rs

@@ -33,9 +33,11 @@ futures.workspace = true
 http.workspace = true
 hyper.workspace = true
 internal-dns.workspace = true
+kstat-rs.workspace = true
 lazy_static.workspace = true
 nexus-client.workspace = true
 omicron-common.workspace = true
+oximeter-instruments.workspace = true
 oximeter-producer.workspace = true
 oximeter.workspace = true
 ron.workspace = true

bin/propolis-server/Cargo.toml

@@ -12,6 +12,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::serial::Serial;
 use crate::server::{BlockBackendMap, CrucibleBackendMap, DeviceMap};
+use crate::stats::virtual_machine::VirtualMachine;
 use anyhow::{Context, Result};
 use crucible_client_types::VolumeConstructionRequest;
 pub use nexus_client::Client as NexusClient;
@@ -334,7 +335,7 @@ impl<'a> MachineInitializer<'a> {
 
     pub fn initialize_qemu_pvpanic(
         &mut self,
-        uuid: uuid::Uuid,
+        virtual_machine: VirtualMachine,
     ) -> Result<(), anyhow::Error> {
         if let Some(ref spec) = self.spec.devices.qemu_pvpanic {
             if spec.enable_isa {
@@ -346,8 +347,10 @@ impl<'a> MachineInitializer<'a> {
                     .insert(pvpanic.type_name().into(), pvpanic.clone());
 
                 if let Some(ref registry) = self.producer_registry {
-                    let producer =
-                        crate::stats::PvpanicProducer::new(uuid, pvpanic);
+                    let producer = crate::stats::PvpanicProducer::new(
+                        virtual_machine,
+                        pvpanic,
+                    );
                     registry.register_producer(producer).context(
                         "failed to register PVPANIC Oximeter producer",
                     )?;