From c9fb7a6534d47fa40939260c6e64194f6eb77ad0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:08:58 -0800 Subject: [PATCH 01/84] [nexus] Add Affinity/Anti-Affinity Groups to API (unimplemented) --- common/src/api/external/mod.rs | 54 + nexus/external-api/output/nexus_tags.txt | 21 + nexus/external-api/src/lib.rs | 225 +++ nexus/src/external_api/http_entrypoints.rs | 355 ++++ .../output/uncovered-authz-endpoints.txt | 18 + nexus/types/src/external_api/params.rs | 76 +- nexus/types/src/external_api/views.rs | 24 +- openapi/nexus.json | 1439 ++++++++++++++++- 8 files changed, 2134 insertions(+), 78 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fd5cbe38042..98832487d5a 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -982,6 +982,10 @@ impl JsonSchema for Hostname { pub enum ResourceType { AddressLot, AddressLotBlock, + AffinityGroup, + AffinityGroupMember, + AntiAffinityGroup, + AntiAffinityGroupMember, AllowList, BackgroundTask, BgpConfig, @@ -1312,6 +1316,56 @@ pub enum InstanceAutoRestartPolicy { BestEffort, } +// AFFINITY GROUPS + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AffinityPolicy { + /// If the affinity request cannot be satisfied, allow it anyway. + /// + /// This enables a "best-effort" attempt to satisfy the affinity policy. + Allow, + + /// If the affinity request cannot be satisfied, fail explicitly. + Fail, +} + +/// Describes the scope of affinity for the purposes of co-location. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FailureDomain { + /// Instances are considered co-located if they are on the same sled + Sled, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum AffinityGroupMember { + Instance(Uuid), +} + +impl SimpleIdentity for AffinityGroupMember { + fn id(&self) -> Uuid { + match self { + AffinityGroupMember::Instance(id) => *id, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum AntiAffinityGroupMember { + Instance(Uuid), +} + +impl SimpleIdentity for AntiAffinityGroupMember { + fn id(&self) -> Uuid { + match self { + AntiAffinityGroupMember::Instance(id) => *id, + } + } +} + // DISKS /// View of a Disk diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 82767e399c0..59ba14e5fb8 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -1,3 +1,24 @@ +API operations found with tag "affinity" +OPERATION ID METHOD URL PATH +affinity_group_create POST /v1/affinity-groups +affinity_group_delete DELETE /v1/affinity-groups/{affinity_group} +affinity_group_list GET /v1/affinity-groups +affinity_group_member_instance_add POST /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_instance_delete DELETE /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_instance_view GET /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_list GET /v1/affinity-groups/{affinity_group}/members +affinity_group_update PUT /v1/affinity-groups/{affinity_group} +affinity_group_view GET /v1/affinity-groups/{affinity_group} +anti_affinity_group_create POST /v1/anti-affinity-groups +anti_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group} +anti_affinity_group_list GET /v1/anti-affinity-groups +anti_affinity_group_member_instance_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_instance_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_instance_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_list GET /v1/anti-affinity-groups/{anti_affinity_group}/members +anti_affinity_group_update PUT /v1/anti-affinity-groups/{anti_affinity_group} +anti_affinity_group_view GET /v1/anti-affinity-groups/{anti_affinity_group} + API operations found with tag "disks" OPERATION ID METHOD URL PATH disk_bulk_write_import POST /v1/disks/{disk}/bulk-write diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index a832bde3199..7daa5514619 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -74,6 +74,13 @@ const PUT_UPDATE_REPOSITORY_MAX_BYTES: usize = 4 * GIB; allow_other_tags = false, policy = EndpointTagPolicy::ExactlyOne, tags = { + "affinity" = { + description = "Affinity groups give control over instance placement.", + external_docs = { + url = "http://docs.oxide.computer/api/affinity" + } + + }, "disks" = { description = "Virtual disks are used to store instance-local data which includes the operating system.", external_docs = { @@ -1257,6 +1264,224 @@ pub trait NexusExternalApi { disk_to_detach: TypedBody, ) -> Result, HttpError>; + // Affinity Groups + + /// List affinity groups + #[endpoint { + method = GET, + path = "/v1/affinity-groups", + tags = ["affinity"], + }] + async fn affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch an affinity group + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// List members of an affinity group + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}/members", + tags = ["affinity"], + }] + async fn affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError>; + + /// Fetch an affinity group member + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an affinity group + #[endpoint { + method = POST, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an affinity group + #[endpoint { + method = DELETE, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Create an affinity group + #[endpoint { + method = POST, + path = "/v1/affinity-groups", + tags = ["affinity"], + }] + async fn affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError>; + + /// Update an affinity group + #[endpoint { + method = PUT, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError>; + + /// Delete an affinity group + #[endpoint { + method = DELETE, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// List anti-affinity groups + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups", + tags = ["affinity"], + }] + async fn anti_affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch an anti-affinity group + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// List members of an anti-affinity group + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError>; + + /// Fetch an anti-affinity group member + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an anti-affinity group + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an anti-affinity group + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Create an anti-affinity group + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups", + tags = ["affinity"], + }] + async fn anti_affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError>; + + /// Update an anti-affinity group + #[endpoint { + method = PUT, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError>; + + /// Delete an anti-affinity group + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + // Certificates /// List certificates for external endpoints diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 191c304216d..906684e7825 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -13,6 +13,7 @@ use super::{ Utilization, Vpc, VpcRouter, VpcSubnet, }, }; +use crate::app::Unimpl; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; use crate::context::ApiContext; @@ -69,7 +70,9 @@ use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::AddressLot; use omicron_common::api::external::AddressLotBlock; use omicron_common::api::external::AddressLotCreateResponse; +use omicron_common::api::external::AffinityGroupMember; use omicron_common::api::external::AggregateBgpMessageHistory; +use omicron_common::api::external::AntiAffinityGroupMember; use omicron_common::api::external::BgpAnnounceSet; use omicron_common::api::external::BgpAnnouncement; use omicron_common::api::external::BgpConfig; @@ -2506,6 +2509,358 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + // Affinity Groups + + async fn affinity_group_list( + rqctx: RequestContext, + _query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_list( + rqctx: RequestContext, + _query_params: Query>, + _path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_add( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_create( + rqctx: RequestContext, + _query_params: Query, + _new_affinity_group_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_update( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + _updated_group: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_list( + rqctx: RequestContext, + _query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_list( + rqctx: RequestContext, + _query_params: Query>, + _path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_add( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_create( + rqctx: RequestContext, + _query_params: Query, + _new_anti_affinity_group_params: TypedBody< + params::AntiAffinityGroupCreate, + >, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_update( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + _updated_group: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + // Certificates async fn certificate_list( diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index 8a639f1224c..c943728beb4 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -1,10 +1,22 @@ API endpoints with no coverage in authz tests: probe_delete (delete "/experimental/v1/probes/{probe}") +affinity_group_delete (delete "/v1/affinity-groups/{affinity_group}") +affinity_group_member_instance_delete (delete "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") +anti_affinity_group_delete (delete "/v1/anti-affinity-groups/{anti_affinity_group}") +anti_affinity_group_member_instance_delete (delete "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") probe_list (get "/experimental/v1/probes") probe_view (get "/experimental/v1/probes/{probe}") support_bundle_download (get "/experimental/v1/system/support-bundles/{support_bundle}/download") support_bundle_download_file (get "/experimental/v1/system/support-bundles/{support_bundle}/download/{file}") support_bundle_index (get "/experimental/v1/system/support-bundles/{support_bundle}/index") +affinity_group_list (get "/v1/affinity-groups") +affinity_group_view (get "/v1/affinity-groups/{affinity_group}") +affinity_group_member_list (get "/v1/affinity-groups/{affinity_group}/members") +affinity_group_member_instance_view (get "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") +anti_affinity_group_list (get "/v1/anti-affinity-groups") +anti_affinity_group_view (get "/v1/anti-affinity-groups/{anti_affinity_group}") +anti_affinity_group_member_list (get "/v1/anti-affinity-groups/{anti_affinity_group}/members") +anti_affinity_group_member_instance_view (get "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") ping (get "/v1/ping") networking_switch_port_lldp_neighbors (get "/v1/system/hardware/rack-switch-port/{rack_id}/{switch_location}/{port}/lldp/neighbors") networking_switch_port_lldp_config_view (get "/v1/system/hardware/switch-port/{port}/lldp/config") @@ -16,6 +28,12 @@ device_auth_confirm (post "/device/confirm") device_access_token (post "/device/token") probe_create (post "/experimental/v1/probes") login_saml (post "/login/{silo_name}/saml/{provider_name}") +affinity_group_create (post "/v1/affinity-groups") +affinity_group_member_instance_add (post "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") +anti_affinity_group_create (post "/v1/anti-affinity-groups") +anti_affinity_group_member_instance_add (post "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") login_local (post "/v1/login/{silo_name}/local") logout (post "/v1/logout") networking_switch_port_lldp_config_update (post "/v1/system/hardware/switch-port/{port}/lldp/config") +affinity_group_update (put "/v1/affinity-groups/{affinity_group}") +anti_affinity_group_update (put "/v1/anti-affinity-groups/{anti_affinity_group}") diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 8294e23e2f2..e87acd4e590 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -10,11 +10,11 @@ use base64::Engine; use chrono::{DateTime, Utc}; use http::Uri; use omicron_common::api::external::{ - AddressLotKind, AllowedSourceIps, BfdMode, BgpPeer, ByteCount, Hostname, - IdentityMetadataCreateParams, IdentityMetadataUpdateParams, - InstanceAutoRestartPolicy, InstanceCpuCount, LinkFec, LinkSpeed, Name, - NameOrId, PaginationOrder, RouteDestination, RouteTarget, SemverVersion, - TxEqConfig, UserId, + AddressLotKind, AffinityPolicy, AllowedSourceIps, BfdMode, BgpPeer, + ByteCount, FailureDomain, Hostname, IdentityMetadataCreateParams, + IdentityMetadataUpdateParams, InstanceAutoRestartPolicy, InstanceCpuCount, + LinkFec, LinkSpeed, Name, NameOrId, PaginationOrder, RouteDestination, + RouteTarget, SemverVersion, TxEqConfig, UserId, }; use omicron_common::disk::DiskVariant; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; @@ -69,6 +69,8 @@ pub struct UninitializedSledId { pub part: String, } +path_param!(AffinityGroupPath, affinity_group, "affinity group"); +path_param!(AntiAffinityGroupPath, anti_affinity_group, "anti affinity group"); path_param!(ProjectPath, project, "project"); path_param!(InstancePath, instance, "instance"); path_param!(NetworkInterfacePath, interface, "network interface"); @@ -806,6 +808,70 @@ where Ok(v) } +// AFFINITY GROUPS + +/// Create-time parameters for an `AffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroupCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +/// Updateable properties of an `AffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroupUpdate { + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AffinityInstanceGroupMemberPath { + pub affinity_group: NameOrId, + pub instance: NameOrId, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AntiAffinityInstanceGroupMemberPath { + pub anti_affinity_group: NameOrId, + pub instance: NameOrId, +} + +/// Create-time parameters for an `AntiAffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroupCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +/// Updateable properties of an `AntiAffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroupUpdate { + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, +} + +#[derive(Deserialize, JsonSchema, Clone)] +pub struct AffinityGroupSelector { + /// Name or ID of the project, only required if `affinity_group` is provided as a `Name` + pub project: Option, + /// Name or ID of the Affinity Group + pub affinity_group: NameOrId, +} + +#[derive(Deserialize, JsonSchema, Clone)] +pub struct AntiAffinityGroupSelector { + /// Name or ID of the project, only required if `anti_affinity_group` is provided as a `Name` + pub project: Option, + /// Name or ID of the Anti Affinity Group + pub anti_affinity_group: NameOrId, +} + // PROJECTS /// Create-time parameters for a `Project` diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 3430d06f724..fe93e4cae60 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -13,9 +13,9 @@ use chrono::DateTime; use chrono::Utc; use diffus::Diffus; use omicron_common::api::external::{ - AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, Digest, Error, - IdentityMetadata, InstanceState, Name, ObjectIdentity, RoleName, - SimpleIdentityOrName, + AffinityPolicy, AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, + Digest, Error, FailureDomain, IdentityMetadata, InstanceState, Name, + ObjectIdentity, RoleName, SimpleIdentityOrName, }; use oxnet::{Ipv4Net, Ipv6Net}; use schemars::JsonSchema; @@ -112,6 +112,24 @@ impl SimpleIdentityOrName for SiloUtilization { } } +// AFFINITY GROUPS + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroup { + #[serde(flatten)] + pub identity: IdentityMetadata, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroup { + #[serde(flatten)] + pub identity: IdentityMetadata, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + // IDENTITY PROVIDER #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] diff --git a/openapi/nexus.json b/openapi/nexus.json index 45f2d4787cf..2e4ef3ac597 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -683,6 +683,958 @@ } } }, + "/v1/affinity-groups": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List affinity groups", + "operationId": "affinity_group_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Create an affinity group", + "operationId": "affinity_group_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/affinity-groups/{affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an affinity group", + "operationId": "affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "affinity" + ], + "summary": "Update an affinity group", + "operationId": "affinity_group_update", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Delete an affinity group", + "operationId": "affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/affinity-groups/{affinity_group}/members": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List members of an affinity group", + "operationId": "affinity_group_member_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMemberResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/affinity-groups/{affinity_group}/members/instance/{instance}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an affinity group member", + "operationId": "affinity_group_member_instance_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an affinity group", + "operationId": "affinity_group_member_instance_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an affinity group", + "operationId": "affinity_group_member_instance_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List anti-affinity groups", + "operationId": "anti_affinity_group_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Create an anti-affinity group", + "operationId": "anti_affinity_group_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group", + "operationId": "anti_affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "affinity" + ], + "summary": "Update an anti-affinity group", + "operationId": "anti_affinity_group_update", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Delete an anti-affinity group", + "operationId": "anti_affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List members of an anti-affinity group", + "operationId": "anti_affinity_group_member_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMemberResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group member", + "operationId": "anti_affinity_group_member_instance_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/certificates": { "get": { "tags": [ @@ -11055,13 +12007,227 @@ "type": "string", "format": "ip" } - }, - "required": [ - "first_address", - "last_address" + }, + "required": [ + "first_address", + "last_address" + ] + }, + "AddressLotBlockResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlock" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AddressLotCreate": { + "description": "Parameters for creating an address lot.", + "type": "object", + "properties": { + "blocks": { + "description": "The blocks to add along with the new address lot.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlockCreate" + } + }, + "description": { + "type": "string" + }, + "kind": { + "description": "The kind of address lot to create.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLotKind" + } + ] + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "blocks", + "description", + "kind", + "name" + ] + }, + "AddressLotCreateResponse": { + "description": "An address lot and associated blocks resulting from creating an address lot.", + "type": "object", + "properties": { + "blocks": { + "description": "The address lot blocks that were created.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlock" + } + }, + "lot": { + "description": "The address lot that was created.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLot" + } + ] + } + }, + "required": [ + "blocks", + "lot" + ] + }, + "AddressLotKind": { + "description": "The kind associated with an address lot.", + "oneOf": [ + { + "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "type": "string", + "enum": [ + "infra" + ] + }, + { + "description": "Pool address lots are used by IP pools.", + "type": "string", + "enum": [ + "pool" + ] + } + ] + }, + "AddressLotResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLot" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AffinityGroup": { + "description": "Identity-related metadata that's included in nearly all public API objects", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "failure_domain", + "id", + "name", + "policy", + "time_created", + "time_modified" + ] + }, + "AffinityGroupCreate": { + "description": "Create-time parameters for an `AffinityGroup`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + } + }, + "required": [ + "description", + "failure_domain", + "name", + "policy" + ] + }, + "AffinityGroupMember": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } ] }, - "AddressLotBlockResultsPage": { + "AffinityGroupMemberResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -11069,7 +12235,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLotBlock" + "$ref": "#/components/schemas/AffinityGroupMember" } }, "next_page": { @@ -11082,104 +12248,63 @@ "items" ] }, - "AddressLotCreate": { - "description": "Parameters for creating an address lot.", + "AffinityGroupResultsPage": { + "description": "A single page of results", "type": "object", "properties": { - "blocks": { - "description": "The blocks to add along with the new address lot.", + "items": { + "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLotBlockCreate" + "$ref": "#/components/schemas/AffinityGroup" } }, - "description": { + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", "type": "string" - }, - "kind": { - "description": "The kind of address lot to create.", - "allOf": [ - { - "$ref": "#/components/schemas/AddressLotKind" - } - ] - }, - "name": { - "$ref": "#/components/schemas/Name" } }, "required": [ - "blocks", - "description", - "kind", - "name" + "items" ] }, - "AddressLotCreateResponse": { - "description": "An address lot and associated blocks resulting from creating an address lot.", + "AffinityGroupUpdate": { + "description": "Updateable properties of an `AffinityGroup`", "type": "object", "properties": { - "blocks": { - "description": "The address lot blocks that were created.", - "type": "array", - "items": { - "$ref": "#/components/schemas/AddressLotBlock" - } + "description": { + "nullable": true, + "type": "string" }, - "lot": { - "description": "The address lot that was created.", + "name": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/Name" } ] } - }, - "required": [ - "blocks", - "lot" - ] + } }, - "AddressLotKind": { - "description": "The kind associated with an address lot.", + "AffinityPolicy": { "oneOf": [ { - "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "description": "If the affinity request cannot be satisfied, allow it anyway.\n\nThis enables a \"best-effort\" attempt to satisfy the affinity policy.", "type": "string", "enum": [ - "infra" + "allow" ] }, { - "description": "Pool address lots are used by IP pools.", + "description": "If the affinity request cannot be satisfied, fail explicitly.", "type": "string", "enum": [ - "pool" + "fail" ] } ] }, - "AddressLotResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/AddressLot" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, "AggregateBgpMessageHistory": { "description": "BGP message history for rack switches.", "type": "object", @@ -11284,6 +12409,161 @@ } ] }, + "AntiAffinityGroup": { + "description": "Identity-related metadata that's included in nearly all public API objects", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "failure_domain", + "id", + "name", + "policy", + "time_created", + "time_modified" + ] + }, + "AntiAffinityGroupCreate": { + "description": "Create-time parameters for an `AntiAffinityGroup`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + } + }, + "required": [ + "description", + "failure_domain", + "name", + "policy" + ] + }, + "AntiAffinityGroupMember": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "AntiAffinityGroupMemberResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AntiAffinityGroupResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AntiAffinityGroupUpdate": { + "description": "Updateable properties of an `AntiAffinityGroup`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "AuthzScope": { "description": "Authorization scope for a timeseries.\n\nThis describes the level at which a user must be authorized to read data from a timeseries. For example, fleet-scoping means the data is only visible to an operator or fleet reader. Project-scoped, on the other hand, indicates that a user will see data limited to the projects on which they have read permissions.", "oneOf": [ @@ -14635,6 +15915,18 @@ "items" ] }, + "FailureDomain": { + "description": "Describes the scope of affinity for the purposes of co-location.", + "oneOf": [ + { + "description": "Instances are considered co-located if they are on the same sled", + "type": "string", + "enum": [ + "sled" + ] + } + ] + }, "FieldSchema": { "description": "The name and type information for a field of a timeseries schema.", "type": "object", @@ -23264,6 +24556,13 @@ } }, "tags": [ + { + "name": "affinity", + "description": "Affinity groups give control over instance placement.", + "externalDocs": { + "url": "http://docs.oxide.computer/api/affinity" + } + }, { "name": "disks", "description": "Virtual disks are used to store instance-local data which includes the operating system.", From 4020517ef2707fd92978760a990f6fe20b1bdaa2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:19:13 -0800 Subject: [PATCH 02/84] [nexus] Add Affinity/Anti-Affinity groups to database --- dev-tools/omdb/src/bin/omdb/db.rs | 1 + nexus/db-model/src/affinity.rs | 260 ++++++++++++++++++ nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/project.rs | 24 +- nexus/db-model/src/schema.rs | 49 +++- nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/sled_resource.rs | 32 ++- nexus/db-queries/src/db/datastore/sled.rs | 26 +- .../background/tasks/abandoned_vmm_reaper.rs | 6 +- nexus/src/app/sagas/instance_common.rs | 13 +- nexus/src/app/sagas/instance_migrate.rs | 3 +- nexus/src/app/sagas/instance_start.rs | 3 +- nexus/src/app/sled.rs | 10 +- nexus/src/external_api/http_entrypoints.rs | 2 +- nexus/tests/integration_tests/schema.rs | 5 +- schema/crdb/affinity/up01.sql | 8 + schema/crdb/affinity/up02.sql | 5 + schema/crdb/affinity/up03.sql | 13 + schema/crdb/affinity/up04.sql | 7 + schema/crdb/affinity/up05.sql | 7 + schema/crdb/affinity/up06.sql | 4 + schema/crdb/affinity/up07.sql | 13 + schema/crdb/affinity/up08.sql | 6 + schema/crdb/affinity/up09.sql | 7 + schema/crdb/affinity/up10.sql | 4 + schema/crdb/affinity/up11.sql | 1 + schema/crdb/affinity/up12.sql | 4 + schema/crdb/dbinit.sql | 104 ++++++- uuid-kinds/src/lib.rs | 2 + 29 files changed, 579 insertions(+), 45 deletions(-) create mode 100644 nexus/db-model/src/affinity.rs create mode 100644 schema/crdb/affinity/up01.sql create mode 100644 schema/crdb/affinity/up02.sql create mode 100644 schema/crdb/affinity/up03.sql create mode 100644 schema/crdb/affinity/up04.sql create mode 100644 schema/crdb/affinity/up05.sql create mode 100644 schema/crdb/affinity/up06.sql create mode 100644 schema/crdb/affinity/up07.sql create mode 100644 schema/crdb/affinity/up08.sql create mode 100644 schema/crdb/affinity/up09.sql create mode 100644 schema/crdb/affinity/up10.sql create mode 100644 schema/crdb/affinity/up11.sql create mode 100644 schema/crdb/affinity/up12.sql diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index bfd7c594a3b..c5f8a1f26c5 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -6191,6 +6191,7 @@ async fn cmd_db_vmm_info( rss_ram: ByteCount(rss), reservoir_ram: ByteCount(reservoir), }, + instance_id: _, } = resource; const SLED_ID: &'static str = "sled ID"; const THREADS: &'static str = "hardware threads"; diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs new file mode 100644 index 00000000000..df97e7d1633 --- /dev/null +++ b/nexus/db-model/src/affinity.rs @@ -0,0 +1,260 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/5.0/. + +// Copyright 2024 Oxide Computer Company + +//! Database representation of affinity and anti-affinity groups + +use super::impl_enum_type; +use super::Name; +use crate::schema::affinity_group; +use crate::schema::affinity_group_instance_membership; +use crate::schema::anti_affinity_group; +use crate::schema::anti_affinity_group_instance_membership; +use crate::typed_uuid::DbTypedUuid; +use chrono::{DateTime, Utc}; +use db_macros::Resource; +use nexus_types::external_api::params; +use nexus_types::external_api::views; +use omicron_common::api::external; +use omicron_common::api::external::IdentityMetadata; +use omicron_uuid_kinds::AffinityGroupKind; +use omicron_uuid_kinds::AffinityGroupUuid; +use omicron_uuid_kinds::AntiAffinityGroupKind; +use omicron_uuid_kinds::AntiAffinityGroupUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceKind; +use omicron_uuid_kinds::InstanceUuid; +use uuid::Uuid; + +impl_enum_type!( + #[derive(SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "affinity_policy", schema = "public"))] + pub struct AffinityPolicyEnum; + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Ord, PartialOrd)] + #[diesel(sql_type = AffinityPolicyEnum)] + pub enum AffinityPolicy; + + // Enum values + Fail => b"fail" + Allow => b"allow" +); + +impl From for external::AffinityPolicy { + fn from(policy: AffinityPolicy) -> Self { + match policy { + AffinityPolicy::Fail => Self::Fail, + AffinityPolicy::Allow => Self::Allow, + } + } +} + +impl From for AffinityPolicy { + fn from(policy: external::AffinityPolicy) -> Self { + match policy { + external::AffinityPolicy::Fail => Self::Fail, + external::AffinityPolicy::Allow => Self::Allow, + } + } +} + +impl_enum_type!( + #[derive(SqlType, Debug)] + #[diesel(postgres_type(name = "failure_domain", schema = "public"))] + pub struct FailureDomainEnum; + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq)] + #[diesel(sql_type = FailureDomainEnum)] + pub enum FailureDomain; + + // Enum values + Sled => b"sled" +); + +impl From for external::FailureDomain { + fn from(domain: FailureDomain) -> Self { + match domain { + FailureDomain::Sled => Self::Sled, + } + } +} + +impl From for FailureDomain { + fn from(domain: external::FailureDomain) -> Self { + match domain { + external::FailureDomain::Sled => Self::Sled, + } + } +} + +#[derive( + Queryable, Insertable, Clone, Debug, Resource, Selectable, PartialEq, +)] +#[diesel(table_name = affinity_group)] +pub struct AffinityGroup { + #[diesel(embed)] + pub identity: AffinityGroupIdentity, + pub project_id: Uuid, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +impl AffinityGroup { + pub fn new(project_id: Uuid, params: params::AffinityGroupCreate) -> Self { + Self { + identity: AffinityGroupIdentity::new( + Uuid::new_v4(), + params.identity, + ), + project_id, + policy: params.policy.into(), + failure_domain: params.failure_domain.into(), + } + } +} + +impl From for views::AffinityGroup { + fn from(group: AffinityGroup) -> Self { + let identity = IdentityMetadata { + id: group.identity.id, + name: group.identity.name.into(), + description: group.identity.description, + time_created: group.identity.time_created, + time_modified: group.identity.time_modified, + }; + Self { + identity, + policy: group.policy.into(), + failure_domain: group.failure_domain.into(), + } + } +} + +/// Describes a set of updates for the [`AffinityGroup`] model. +#[derive(AsChangeset)] +#[diesel(table_name = affinity_group)] +pub struct AffinityGroupUpdate { + pub name: Option, + pub description: Option, + pub time_modified: DateTime, +} + +impl From for AffinityGroupUpdate { + fn from(params: params::AffinityGroupUpdate) -> Self { + Self { + name: params.identity.name.map(Name), + description: params.identity.description, + time_modified: Utc::now(), + } + } +} + +#[derive( + Queryable, Insertable, Clone, Debug, Resource, Selectable, PartialEq, +)] +#[diesel(table_name = anti_affinity_group)] +pub struct AntiAffinityGroup { + #[diesel(embed)] + identity: AntiAffinityGroupIdentity, + pub project_id: Uuid, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +impl AntiAffinityGroup { + pub fn new( + project_id: Uuid, + params: params::AntiAffinityGroupCreate, + ) -> Self { + Self { + identity: AntiAffinityGroupIdentity::new( + Uuid::new_v4(), + params.identity, + ), + project_id, + policy: params.policy.into(), + failure_domain: params.failure_domain.into(), + } + } +} + +impl From for views::AntiAffinityGroup { + fn from(group: AntiAffinityGroup) -> Self { + let identity = IdentityMetadata { + id: group.identity.id, + name: group.identity.name.into(), + description: group.identity.description, + time_created: group.identity.time_created, + time_modified: group.identity.time_modified, + }; + Self { + identity, + policy: group.policy.into(), + failure_domain: group.failure_domain.into(), + } + } +} + +/// Describes a set of updates for the [`AntiAffinityGroup`] model. +#[derive(AsChangeset)] +#[diesel(table_name = anti_affinity_group)] +pub struct AntiAffinityGroupUpdate { + pub name: Option, + pub description: Option, + pub time_modified: DateTime, +} + +impl From for AntiAffinityGroupUpdate { + fn from(params: params::AntiAffinityGroupUpdate) -> Self { + Self { + name: params.identity.name.map(Name), + description: params.identity.description, + time_modified: Utc::now(), + } + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = affinity_group_instance_membership)] +pub struct AffinityGroupInstanceMembership { + pub group_id: DbTypedUuid, + pub instance_id: DbTypedUuid, +} + +impl AffinityGroupInstanceMembership { + pub fn new(group_id: AffinityGroupUuid, instance_id: InstanceUuid) -> Self { + Self { group_id: group_id.into(), instance_id: instance_id.into() } + } +} + +impl From for external::AffinityGroupMember { + fn from(member: AffinityGroupInstanceMembership) -> Self { + Self::Instance(member.instance_id.into_untyped_uuid()) + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = anti_affinity_group_instance_membership)] +pub struct AntiAffinityGroupInstanceMembership { + pub group_id: DbTypedUuid, + pub instance_id: DbTypedUuid, +} + +impl AntiAffinityGroupInstanceMembership { + pub fn new( + group_id: AntiAffinityGroupUuid, + instance_id: InstanceUuid, + ) -> Self { + Self { group_id: group_id.into(), instance_id: instance_id.into() } + } +} + +impl From + for external::AntiAffinityGroupMember +{ + fn from(member: AntiAffinityGroupInstanceMembership) -> Self { + Self::Instance(member.instance_id.into_untyped_uuid()) + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bec35c233c6..939388174e8 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -10,6 +10,7 @@ extern crate diesel; extern crate newtype_derive; mod address_lot; +mod affinity; mod allow_list; mod bfd; mod bgp; @@ -130,6 +131,7 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; pub use address_lot::*; +pub use affinity::*; pub use allow_list::*; pub use bfd::*; pub use bgp::*; diff --git a/nexus/db-model/src/project.rs b/nexus/db-model/src/project.rs index 900a13e9c54..2079751a1c4 100644 --- a/nexus/db-model/src/project.rs +++ b/nexus/db-model/src/project.rs @@ -2,9 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::{Disk, Generation, Instance, Name, Snapshot, Vpc}; +use super::{ + AffinityGroup, AntiAffinityGroup, Disk, Generation, Instance, Name, + Snapshot, Vpc, +}; use crate::collection::DatastoreCollectionConfig; -use crate::schema::{disk, image, instance, project, snapshot, vpc}; +use crate::schema::{ + affinity_group, anti_affinity_group, disk, image, instance, project, + snapshot, vpc, +}; use crate::Image; use chrono::{DateTime, Utc}; use db_macros::Resource; @@ -69,6 +75,20 @@ impl DatastoreCollectionConfig for Project { type CollectionIdColumn = instance::dsl::project_id; } +impl DatastoreCollectionConfig for Project { + type CollectionId = Uuid; + type GenerationNumberColumn = project::dsl::rcgen; + type CollectionTimeDeletedColumn = project::dsl::time_deleted; + type CollectionIdColumn = affinity_group::dsl::project_id; +} + +impl DatastoreCollectionConfig for Project { + type CollectionId = Uuid; + type GenerationNumberColumn = project::dsl::rcgen; + type CollectionTimeDeletedColumn = project::dsl::time_deleted; + type CollectionIdColumn = anti_affinity_group::dsl::project_id; +} + impl DatastoreCollectionConfig for Project { type CollectionId = Uuid; type GenerationNumberColumn = project::dsl::rcgen; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index f734d1e88f3..8881d7dd0ed 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -468,6 +468,48 @@ table! { } } +table! { + affinity_group (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + project_id -> Uuid, + policy -> crate::AffinityPolicyEnum, + failure_domain -> crate::FailureDomainEnum, + } +} + +table! { + anti_affinity_group (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + project_id -> Uuid, + policy -> crate::AffinityPolicyEnum, + failure_domain -> crate::FailureDomainEnum, + } +} + +table! { + affinity_group_instance_membership (group_id, instance_id) { + group_id -> Uuid, + instance_id -> Uuid, + } +} + +table! { + anti_affinity_group_instance_membership (group_id, instance_id) { + group_id -> Uuid, + instance_id -> Uuid, + } +} + table! { metric_producer (id) { id -> Uuid, @@ -915,10 +957,11 @@ table! { sled_resource (id) { id -> Uuid, sled_id -> Uuid, - kind -> crate::SledResourceKindEnum, hardware_threads -> Int8, rss_ram -> Int8, reservoir_ram -> Int8, + kind -> crate::SledResourceKindEnum, + instance_id -> Nullable, } } @@ -2025,6 +2068,10 @@ allow_tables_to_appear_in_same_query!( allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( + anti_affinity_group, + anti_affinity_group_instance_membership, + affinity_group, + affinity_group_instance_membership, bp_omicron_zone, bp_target, rendezvous_debug_dataset, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 150f4d092df..852de12f4c5 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(122, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(123, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(123, "affinity"), KnownVersion::new(122, "tuf-artifact-replication"), KnownVersion::new(121, "dataset-to-crucible-dataset"), KnownVersion::new(120, "rendezvous-debug-dataset"), diff --git a/nexus/db-model/src/sled_resource.rs b/nexus/db-model/src/sled_resource.rs index 27471f4d7a9..de5833aa41f 100644 --- a/nexus/db-model/src/sled_resource.rs +++ b/nexus/db-model/src/sled_resource.rs @@ -3,9 +3,19 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::schema::sled_resource; +use crate::typed_uuid::DbTypedUuid; use crate::{ByteCount, SledResourceKind, SqlU32}; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceKind; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::SledKind; +use omicron_uuid_kinds::SledUuid; use uuid::Uuid; +type DbInstanceUuid = DbTypedUuid; +type DbSledUuid = DbTypedUuid; + #[derive(Clone, Selectable, Queryable, Insertable, Debug)] #[diesel(table_name = sled_resource)] pub struct Resources { @@ -29,20 +39,28 @@ impl Resources { #[diesel(table_name = sled_resource)] pub struct SledResource { pub id: Uuid, - pub sled_id: Uuid, - pub kind: SledResourceKind, + pub sled_id: DbSledUuid, #[diesel(embed)] pub resources: Resources, + + pub kind: SledResourceKind, + pub instance_id: Option, } impl SledResource { - pub fn new( - id: Uuid, - sled_id: Uuid, - kind: SledResourceKind, + pub fn new_for_vmm( + id: PropolisUuid, + instance_id: InstanceUuid, + sled_id: SledUuid, resources: Resources, ) -> Self { - Self { id, sled_id, kind, resources } + Self { + id: id.into_untyped_uuid(), + instance_id: Some(instance_id.into()), + sled_id: sled_id.into(), + kind: SledResourceKind::Instance, + resources, + } } } diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 87d5c324c63..ee4e6c31cf7 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -40,6 +40,8 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; use std::fmt; use strum::IntoEnumIterator; @@ -185,8 +187,8 @@ impl DataStore { pub async fn sled_reservation_create( &self, opctx: &OpContext, - resource_id: Uuid, - resource_kind: db::model::SledResourceKind, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> CreateResult { @@ -210,7 +212,7 @@ impl DataStore { use db::schema::sled_resource::dsl as resource_dsl; // Check if resource ID already exists - if so, return it. let old_resource = resource_dsl::sled_resource - .filter(resource_dsl::id.eq(resource_id)) + .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) .select(SledResource::as_select()) .limit(1) .load_async(&conn) @@ -309,10 +311,10 @@ impl DataStore { // Create a SledResource record, associate it with the target // sled. - let resource = SledResource::new( - resource_id, - sled_targets[0], - resource_kind, + let resource = SledResource::new_for_vmm( + propolis_id, + instance_id, + SledUuid::from_untyped_uuid(sled_targets[0]), resources, ); @@ -1113,8 +1115,8 @@ pub(in crate::db::datastore) mod test { let error = datastore .sled_reservation_create( &opctx, - Uuid::new_v4(), - db::model::SledResourceKind::Instance, + InstanceUuid::new_v4(), + PropolisUuid::new_v4(), resources.clone(), constraints, ) @@ -1134,15 +1136,15 @@ pub(in crate::db::datastore) mod test { let resource = datastore .sled_reservation_create( &opctx, - Uuid::new_v4(), - db::model::SledResourceKind::Instance, + InstanceUuid::new_v4(), + PropolisUuid::new_v4(), resources.clone(), constraints, ) .await .unwrap(); assert_eq!( - resource.sled_id, + resource.sled_id.into_untyped_uuid(), provisionable_sled.id(), "resource is always allocated to the provisionable sled" ); diff --git a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs index ce232a14113..1853a82fb19 100644 --- a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs +++ b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs @@ -201,12 +201,12 @@ mod tests { use nexus_db_model::Generation; use nexus_db_model::Resources; use nexus_db_model::SledResource; - use nexus_db_model::SledResourceKind; use nexus_db_model::Vmm; use nexus_db_model::VmmRuntimeState; use nexus_db_model::VmmState; use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; + use omicron_uuid_kinds::InstanceUuid; use uuid::Uuid; type ControlPlaneTestContext = @@ -267,8 +267,8 @@ mod tests { dbg!(datastore .sled_reservation_create( &opctx, - destroyed_vmm_id.into_untyped_uuid(), - SledResourceKind::Instance, + InstanceUuid::from_untyped_uuid(instance.identity.id), + destroyed_vmm_id, resources.clone(), constraints, ) diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index 9f9fefea629..22134e8f6d1 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -38,6 +38,7 @@ pub(super) struct VmmAndSledIds { /// `propolis_id`. pub async fn reserve_vmm_resources( nexus: &Nexus, + instance_id: InstanceUuid, propolis_id: PropolisUuid, ncpus: u32, guest_memory: ByteCount, @@ -50,8 +51,8 @@ pub async fn reserve_vmm_resources( // See https://rfd.shared.oxide.computer/rfd/0205 for a more complete // discussion. // - // Right now, allocate an instance to any random sled agent. This has a few - // problems: + // Right now, allocate an instance to any random sled agent, as long as + // "constraints" and affinity rules are respected. This has a few problems: // // - There's no consideration for "health of the sled" here, other than // "time_deleted = Null". If the sled is rebooting, in a known unhealthy @@ -61,10 +62,6 @@ pub async fn reserve_vmm_resources( // - This is selecting a random sled from all sleds in the cluster. For // multi-rack, this is going to fling the sled to an arbitrary system. // Maybe that's okay, but worth knowing about explicitly. - // - // - This doesn't take into account anti-affinity - users will want to - // schedule instances that belong to a cluster on different failure - // domains. See https://github.com/oxidecomputer/omicron/issues/1705. let resources = db::model::Resources::new( ncpus, ByteCount::try_from(0i64).unwrap(), @@ -73,8 +70,8 @@ pub async fn reserve_vmm_resources( let resource = nexus .reserve_on_random_sled( - propolis_id.into_untyped_uuid(), - nexus_db_model::SledResourceKind::Instance, + instance_id, + propolis_id, resources, constraints, ) diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 0789ef7a484..465105a813f 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -186,6 +186,7 @@ async fn sim_reserve_sled_resources( let resource = super::instance_common::reserve_vmm_resources( osagactx.nexus(), + InstanceUuid::from_untyped_uuid(params.instance.id()), propolis_id, u32::from(params.instance.ncpus.0 .0), params.instance.memory, @@ -193,7 +194,7 @@ async fn sim_reserve_sled_resources( ) .await?; - Ok(SledUuid::from_untyped_uuid(resource.sled_id)) + Ok(resource.sled_id.into()) } async fn sim_release_sled_resources( diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index aa09a165c83..e6af6453291 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -163,6 +163,7 @@ async fn sis_alloc_server( let resource = super::instance_common::reserve_vmm_resources( osagactx.nexus(), + InstanceUuid::from_untyped_uuid(params.db_instance.id()), propolis_id, u32::from(hardware_threads.0), reservoir_ram, @@ -170,7 +171,7 @@ async fn sis_alloc_server( ) .await?; - Ok(SledUuid::from_untyped_uuid(resource.sled_id)) + Ok(resource.sled_id.into()) } async fn sis_alloc_server_undo( diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index d58e111b6e6..ffa507ffb8e 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -25,7 +25,9 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PhysicalDiskUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; use sled_agent_client::Client as SledAgentClient; use std::net::SocketAddrV6; @@ -164,16 +166,16 @@ impl super::Nexus { pub(crate) async fn reserve_on_random_sled( &self, - resource_id: Uuid, - resource_kind: db::model::SledResourceKind, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> Result { self.db_datastore .sled_reservation_create( &self.opctx_alloc, - resource_id, - resource_kind, + instance_id, + propolis_id, resources, constraints, ) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 906684e7825..887a6d35a65 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -13,9 +13,9 @@ use super::{ Utilization, Vpc, VpcRouter, VpcSubnet, }, }; -use crate::app::Unimpl; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; +use crate::app::Unimpl; use crate::context::ApiContext; use crate::external_api::shared; use dropshot::Body; diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index fa41d17aa96..cbc041f741a 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -17,6 +17,8 @@ use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::SledUuid; use pretty_assertions::{assert_eq, assert_ne}; use similar_asserts; use slog::Logger; @@ -987,7 +989,8 @@ async fn dbinit_equals_sum_of_all_up() { diesel::insert_into(dsl::sled_resource) .values(SledResource { id: Uuid::new_v4(), - sled_id: Uuid::new_v4(), + instance_id: Some(InstanceUuid::new_v4().into()), + sled_id: SledUuid::new_v4().into(), kind: SledResourceKind::Instance, resources: Resources { hardware_threads: 8_u32.into(), diff --git a/schema/crdb/affinity/up01.sql b/schema/crdb/affinity/up01.sql new file mode 100644 index 00000000000..06e5fb06f94 --- /dev/null +++ b/schema/crdb/affinity/up01.sql @@ -0,0 +1,8 @@ +CREATE TYPE IF NOT EXISTS omicron.public.affinity_policy AS ENUM ( + -- If the affinity request cannot be satisfied, fail. + 'fail', + + -- If the affinity request cannot be satisfied, allow it anyway. + 'allow' +); + diff --git a/schema/crdb/affinity/up02.sql b/schema/crdb/affinity/up02.sql new file mode 100644 index 00000000000..9165e6c4006 --- /dev/null +++ b/schema/crdb/affinity/up02.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.failure_domain AS ENUM ( + -- Instances are co-located if they are on the same sled. + 'sled' +); + diff --git a/schema/crdb/affinity/up03.sql b/schema/crdb/affinity/up03.sql new file mode 100644 index 00000000000..79238402f8d --- /dev/null +++ b/schema/crdb/affinity/up03.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + diff --git a/schema/crdb/affinity/up04.sql b/schema/crdb/affinity/up04.sql new file mode 100644 index 00000000000..3f455884b98 --- /dev/null +++ b/schema/crdb/affinity/up04.sql @@ -0,0 +1,7 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_affinity_group_by_project ON omicron.public.affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + + diff --git a/schema/crdb/affinity/up05.sql b/schema/crdb/affinity/up05.sql new file mode 100644 index 00000000000..87f5f9e9c11 --- /dev/null +++ b/schema/crdb/affinity/up05.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + diff --git a/schema/crdb/affinity/up06.sql b/schema/crdb/affinity/up06.sql new file mode 100644 index 00000000000..f86037bda3f --- /dev/null +++ b/schema/crdb/affinity/up06.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_affinity_group_instance_membership_by_instance ON omicron.public.affinity_group_instance_membership ( + instance_id +); + diff --git a/schema/crdb/affinity/up07.sql b/schema/crdb/affinity/up07.sql new file mode 100644 index 00000000000..0409c1c4c7b --- /dev/null +++ b/schema/crdb/affinity/up07.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Anti-Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + diff --git a/schema/crdb/affinity/up08.sql b/schema/crdb/affinity/up08.sql new file mode 100644 index 00000000000..536fcf35e37 --- /dev/null +++ b/schema/crdb/affinity/up08.sql @@ -0,0 +1,6 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_anti_affinity_group_by_project ON omicron.public.anti_affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + diff --git a/schema/crdb/affinity/up09.sql b/schema/crdb/affinity/up09.sql new file mode 100644 index 00000000000..9b3333c934e --- /dev/null +++ b/schema/crdb/affinity/up09.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + diff --git a/schema/crdb/affinity/up10.sql b/schema/crdb/affinity/up10.sql new file mode 100644 index 00000000000..c57d98310c5 --- /dev/null +++ b/schema/crdb/affinity/up10.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_instance ON omicron.public.anti_affinity_group_instance_membership ( + instance_id +); + diff --git a/schema/crdb/affinity/up11.sql b/schema/crdb/affinity/up11.sql new file mode 100644 index 00000000000..e3b328c326d --- /dev/null +++ b/schema/crdb/affinity/up11.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.sled_resource ADD COLUMN IF NOT EXISTS instance_id UUID; diff --git a/schema/crdb/affinity/up12.sql b/schema/crdb/affinity/up12.sql new file mode 100644 index 00000000000..1b76693bf29 --- /dev/null +++ b/schema/crdb/affinity/up12.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( + instance_id +); + diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1ee7b985aff..a2fa3d48e23 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -217,7 +217,7 @@ CREATE INDEX IF NOT EXISTS lookup_sled_by_policy_and_state ON omicron.public.sle ); CREATE TYPE IF NOT EXISTS omicron.public.sled_resource_kind AS ENUM ( - -- omicron.public.instance + -- omicron.public.vmm ; this is called "instance" for historical reasons. 'instance' -- We expect to other resource kinds here in the future; e.g., to track -- resources used by control plane services. For now, we only track @@ -242,7 +242,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( reservoir_ram INT8 NOT NULL, -- Identifies the type of the resource - kind omicron.public.sled_resource_kind NOT NULL + kind omicron.public.sled_resource_kind NOT NULL, + + -- The UUID of an instance, if this resource belongs to an instance. + instance_id UUID + + -- TODO Add constraint that if kind is instance, instance_id is not NULL? + -- Or will that break backwards compatibility? ); -- Allow looking up all resources which reside on a sled @@ -251,6 +257,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled id ); +-- Allow looking up all resources by instance +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( + instance_id +); -- Table of all sled subnets allocated for sleds added to an already initialized -- rack. The sleds in this table and their allocated subnets are created before @@ -4094,6 +4104,94 @@ CREATE INDEX IF NOT EXISTS lookup_usable_rendezvous_debug_dataset COMMIT; BEGIN; +-- Describes what happens when +-- (for affinity groups) instance cannot be co-located, or +-- (for anti-affinity groups) instance must be co-located, or +CREATE TYPE IF NOT EXISTS omicron.public.affinity_policy AS ENUM ( + -- If the affinity request cannot be satisfied, fail. + 'fail', + + -- If the affinity request cannot be satisfied, allow it anyway. + 'allow' +); + +-- Determines what "co-location" means for instances within an affinity +-- or anti-affinity group. +CREATE TYPE IF NOT EXISTS omicron.public.failure_domain AS ENUM ( + -- Instances are co-located if they are on the same sled. + 'sled' +); + +-- Describes a grouping of related instances that should be co-located. +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + +-- Names for affinity groups within a project should be unique +CREATE UNIQUE INDEX IF NOT EXISTS lookup_affinity_group_by_project ON omicron.public.affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + +-- Describes an instance's membership within an affinity group +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + +-- We need to look up all memberships of an instance so we can revoke these +-- memberships efficiently when instances are deleted. +CREATE INDEX IF NOT EXISTS lookup_affinity_group_instance_membership_by_instance ON omicron.public.affinity_group_instance_membership ( + instance_id +); + +-- Describes a collection of instances that should not be co-located. +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Anti-Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + +-- Names for anti-affinity groups within a project should be unique +CREATE UNIQUE INDEX IF NOT EXISTS lookup_anti_affinity_group_by_project ON omicron.public.anti_affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + +-- Describes an instance's membership within an anti-affinity group +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + +-- We need to look up all memberships of an instance so we can revoke these +-- memberships efficiently when instances are deleted. +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_instance ON omicron.public.anti_affinity_group_instance_membership ( + instance_id +); + -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, @@ -4810,7 +4908,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '122.0.0', NULL) + (TRUE, NOW(), NOW(), '123.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index 1d6dc600226..a27fc43b8e0 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -51,6 +51,8 @@ macro_rules! impl_typed_uuid_kind { // Please keep this list in alphabetical order. impl_typed_uuid_kind! { + AffinityGroup => "affinity_group", + AntiAffinityGroup => "anti_affinity_group", Blueprint => "blueprint", Collection => "collection", Dataset => "dataset", From 8f1d37ceae8373cef2e2b815e26c556655f07b32 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:25:54 -0800 Subject: [PATCH 03/84] [nexus] Add CRUD implementations for Affinity/Anti-Affinity Groups --- nexus/auth/src/authz/api_resources.rs | 16 + nexus/auth/src/authz/oso_generic.rs | 2 + nexus/db-queries/src/db/datastore/affinity.rs | 2701 +++++++++++++++++ nexus/db-queries/src/db/datastore/instance.rs | 7 + nexus/db-queries/src/db/datastore/mod.rs | 1 + nexus/db-queries/src/db/datastore/project.rs | 5 + nexus/db-queries/src/db/lookup.rs | 30 +- 7 files changed, 2761 insertions(+), 1 deletion(-) create mode 100644 nexus/db-queries/src/db/datastore/affinity.rs diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index 745a699cf2b..4d588036b7e 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -713,6 +713,22 @@ authz_resource! { polar_snippet = InProject, } +authz_resource! { + name = "AffinityGroup", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + +authz_resource! { + name = "AntiAffinityGroup", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + authz_resource! { name = "InstanceNetworkInterface", parent = "Instance", diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index 321bb98b1c6..32b3dbd1f80 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -125,6 +125,8 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { Disk::init(), Snapshot::init(), ProjectImage::init(), + AffinityGroup::init(), + AntiAffinityGroup::init(), Instance::init(), IpPool::init(), InstanceNetworkInterface::init(), diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs new file mode 100644 index 00000000000..e09a0e43406 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -0,0 +1,2701 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods on Affinity Groups + +use super::DataStore; +use crate::authz; +use crate::authz::ApiResource; +use crate::db; +use crate::db::collection_insert::AsyncInsertError; +use crate::db::collection_insert::DatastoreCollection; +use crate::db::datastore::OpContext; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::identity::Resource; +use crate::db::model::AffinityGroup; +use crate::db::model::AffinityGroupInstanceMembership; +use crate::db::model::AffinityGroupUpdate; +use crate::db::model::AntiAffinityGroup; +use crate::db::model::AntiAffinityGroupInstanceMembership; +use crate::db::model::AntiAffinityGroupUpdate; +use crate::db::model::InstanceState; +use crate::db::model::Name; +use crate::db::model::Project; +use crate::db::pagination::paginated; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::Utc; +use diesel::prelude::*; +use omicron_common::api::external; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; +use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::AffinityGroupUuid; +use omicron_uuid_kinds::AntiAffinityGroupUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use ref_cast::RefCast; + +impl DataStore { + pub async fn affinity_group_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::affinity_group::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::affinity_group, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::affinity_group, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(AffinityGroup::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn anti_affinity_group_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::anti_affinity_group::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::anti_affinity_group, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::anti_affinity_group, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(AntiAffinityGroup::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn affinity_group_create( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + group: AffinityGroup, + ) -> CreateResult { + use db::schema::affinity_group::dsl; + + opctx.authorize(authz::Action::CreateChild, authz_project).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let name = group.name().as_str().to_string(); + + let affinity_group: AffinityGroup = Project::insert_resource( + authz_project.id(), + diesel::insert_into(dsl::affinity_group).values(group), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => authz_project.not_found(), + AsyncInsertError::DatabaseError(diesel_error) => { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict(ResourceType::AffinityGroup, &name), + ) + } + })?; + Ok(affinity_group) + } + + pub async fn anti_affinity_group_create( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + group: AntiAffinityGroup, + ) -> CreateResult { + use db::schema::anti_affinity_group::dsl; + + opctx.authorize(authz::Action::CreateChild, authz_project).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let name = group.name().as_str().to_string(); + + let anti_affinity_group: AntiAffinityGroup = Project::insert_resource( + authz_project.id(), + diesel::insert_into(dsl::anti_affinity_group).values(group), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => authz_project.not_found(), + AsyncInsertError::DatabaseError(diesel_error) => { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroup, + &name, + ), + ) + } + })?; + Ok(anti_affinity_group) + } + + pub async fn affinity_group_update( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + updates: AffinityGroupUpdate, + ) -> UpdateResult { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + use db::schema::affinity_group::dsl; + diesel::update(dsl::affinity_group) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_affinity_group.id())) + .set(updates) + .returning(AffinityGroup::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_affinity_group), + ) + }) + } + + pub async fn affinity_group_delete( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_affinity_group).await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::affinity_group::dsl as group_dsl; + let now = Utc::now(); + + // Delete the Affinity Group + diesel::update(group_dsl::affinity_group) + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .set(group_dsl::time_deleted.eq(now)) + .returning(AffinityGroup::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Ensure all memberships in the affinity group are deleted + use db::schema::affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_update( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + updates: AntiAffinityGroupUpdate, + ) -> UpdateResult { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + use db::schema::anti_affinity_group::dsl; + diesel::update(dsl::anti_affinity_group) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_anti_affinity_group.id())) + .set(updates) + .returning(AntiAffinityGroup::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_anti_affinity_group), + ) + }) + } + + pub async fn anti_affinity_group_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + ) -> DeleteResult { + opctx + .authorize(authz::Action::Delete, authz_anti_affinity_group) + .await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::anti_affinity_group::dsl as group_dsl; + let now = Utc::now(); + + // Delete the Anti Affinity Group + diesel::update(group_dsl::anti_affinity_group) + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .set(group_dsl::time_deleted.eq(now)) + .returning(AntiAffinityGroup::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Ensure all memberships in the anti affinity group are deleted + use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn affinity_group_member_list( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, authz_affinity_group).await?; + + use db::schema::affinity_group_instance_membership::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => paginated( + dsl::affinity_group_instance_membership, + dsl::instance_id, + &pagparams, + ), + PaginatedBy::Name(_) => { + return Err(Error::invalid_request( + "Cannot paginate group members by name", + )); + } + } + .filter(dsl::group_id.eq(authz_affinity_group.id())) + .select(AffinityGroupInstanceMembership::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn anti_affinity_group_member_list( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + + use db::schema::anti_affinity_group_instance_membership::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => paginated( + dsl::anti_affinity_group_instance_membership, + dsl::instance_id, + &pagparams, + ), + PaginatedBy::Name(_) => { + return Err(Error::invalid_request( + "Cannot paginate group members by name", + )); + } + } + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn affinity_group_member_view( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + use db::schema::affinity_group_instance_membership::dsl; + dsl::affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id)) + .select(AffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AffinityGroupMember, + LookupType::by_id(instance_id), + ), + ) + }) + } + + pub async fn anti_affinity_group_member_view( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + use db::schema::anti_affinity_group_instance_membership::dsl; + dsl::anti_affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id)) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id(instance_id), + ), + ) + }) + } + + pub async fn affinity_group_member_add( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_member_add") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::affinity_group::dsl as group_dsl; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists, and has no VMM + // + // NOTE: I'd prefer to use the "LookupPath" infrastructure + // to look up the path, but that API does not give the + // option to use the transaction's database connection. + // + // Looking up the instance on a different database + // connection than the transaction risks several concurrency + // issues, so we do the lookup manually. + let instance_state = instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::state) + .first_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + // NOTE: It may be possible to add non-stopped instances to + // affinity groups, depending on where they have already + // been placed. However, only operating on "stopped" + // instances is much easier to work with, as it does not + // require any understanding of the group policy. + match instance_state { + InstanceState::NoVmm => (), + other => { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to affinity group in state: {other}" + ) + ))); + }, + } + + diesel::insert_into(membership_dsl::affinity_group_instance_membership) + .values(AffinityGroupInstanceMembership::new( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AffinityGroupMember, + &instance_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_member_add( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_add") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists, and has no VMM + let instance_state = instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::state) + .first_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + // NOTE: It may be possible to add non-stopped instances to + // anti affinity groups, depending on where they have already + // been placed. However, only operating on "stopped" + // instances is much easier to work with, as it does not + // require any understanding of the group policy. + match instance_state { + InstanceState::NoVmm => (), + other => { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to anti-affinity group in state: {other}" + ) + ))); + }, + } + + diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) + .values(AntiAffinityGroupInstanceMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroupMember, + &instance_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn instance_affinity_group_memberships_delete( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> Result<(), Error> { + use db::schema::affinity_group_instance_membership::dsl; + + diesel::delete(dsl::affinity_group_instance_membership) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + + pub async fn instance_anti_affinity_group_memberships_delete( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> Result<(), Error> { + use db::schema::anti_affinity_group_instance_membership::dsl; + + diesel::delete(dsl::anti_affinity_group_instance_membership) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + + pub async fn affinity_group_member_delete( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::affinity_group::dsl as group_dsl; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists + instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) + .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) + .filter(membership_dsl::instance_id.eq(instance_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(instance_id).into_not_found( + ResourceType::AffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists + instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) + .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(membership_dsl::instance_id.eq(instance_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(instance_id).into_not_found( + ResourceType::AntiAffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::db::lookup::LookupPath; + use crate::db::pub_test_utils::TestDatabase; + use nexus_db_model::Instance; + use nexus_types::external_api::params; + use omicron_common::api::external::{ + self, ByteCount, DataPageParams, IdentityMetadataCreateParams, + }; + use omicron_test_utils::dev; + use omicron_uuid_kinds::InstanceUuid; + use std::num::NonZeroU32; + + // Helper function for creating a project + async fn create_project( + opctx: &OpContext, + datastore: &DataStore, + name: &str, + ) -> (authz::Project, Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() + } + + // Helper function for creating an affinity group with + // arbitrary configuration. + async fn create_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> CreateResult { + let group = AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + }, + ); + datastore.affinity_group_create(&opctx, &authz_project, group).await + } + + // Helper function for creating an anti-affinity group with + // arbitrary configuration. + async fn create_anti_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> CreateResult { + let group = AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + }, + ); + datastore + .anti_affinity_group_create(&opctx, &authz_project, group) + .await + } + + // Helper function for creating an instance without a VMM. + async fn create_instance_record( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> Instance { + let instance = Instance::new( + InstanceUuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + ncpus: 2i64.try_into().unwrap(), + memory: ByteCount::from_gibibytes_u32(16), + hostname: "myhostname".try_into().unwrap(), + user_data: Vec::new(), + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: Vec::new(), + disks: Vec::new(), + boot_disk: None, + ssh_public_keys: None, + start: false, + auto_restart_policy: Default::default(), + }, + ); + + let instance = datastore + .project_create_instance(&opctx, &authz_project, instance) + .await + .unwrap(); + + set_instance_state_stopped(&datastore, instance.id()).await; + + instance + } + + // Helper for explicitly modifying instance state. + // + // The interaction we typically use to create and modify instance state + // is more complex in production, since it's the result of a back-and-forth + // between Nexus and Sled Agent, using carefully crafted rcgen values. + // + // Here, we just set the value of state explicitly. Be warned, there + // are no guardrails! + async fn set_instance_state_stopped( + datastore: &DataStore, + instance: uuid::Uuid, + ) { + use db::schema::instance::dsl; + diesel::update(dsl::instance) + .filter(dsl::id.eq(instance)) + .set(( + dsl::state.eq(db::model::InstanceState::NoVmm), + dsl::active_propolis_id.eq(None::), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + async fn set_instance_state_running( + datastore: &DataStore, + instance: uuid::Uuid, + ) { + use db::schema::instance::dsl; + diesel::update(dsl::instance) + .filter(dsl::id.eq(instance)) + .set(( + dsl::state.eq(db::model::InstanceState::Vmm), + dsl::active_propolis_id.eq(uuid::Uuid::new_v4()), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn affinity_groups_are_project_scoped() { + // Setup + let logctx = dev::test_setup_log("affinity_groups_are_project_scoped"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (authz_project, _) = + create_project(&opctx, &datastore, "my-project").await; + + let (authz_other_project, _) = + create_project(&opctx, &datastore, "my-other-project").await; + + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + + // To start: No groups exist + let groups = datastore + .affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Create a group + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Now when we list groups, we'll see the one we created. + let groups = datastore + .affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], group); + + // This group won't appear in the other project + let groups = datastore + .affinity_group_list(&opctx, &authz_other_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_groups_are_project_scoped() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_groups_are_project_scoped"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (authz_project, _) = + create_project(&opctx, &datastore, "my-project").await; + + let (authz_other_project, _) = + create_project(&opctx, &datastore, "my-other-project").await; + + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + + // To start: No groups exist + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Create a group + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Now when we list groups, we'll see the one we created. + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], group); + + // This group won't appear in the other project + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_other_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_groups_prevent_project_deletion() { + // Setup + let logctx = + dev::test_setup_log("affinity_groups_prevent_project_deletion"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, mut project) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // If we try to delete the project, we'll fail. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string() + .contains("project to be deleted contains an affinity group"), + "{err:?}" + ); + + // Delete the group, then try to delete the project again. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + // When the group was created, it bumped the rcgen in the project. If we + // have an old view of the project, we expect a "concurrent + // modification" error. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(err.to_string().contains("concurrent modification"), "{err:?}"); + + // If we update rcgen, however, and the group has been deleted, + // we can successfully delete the project. + project.rcgen = project.rcgen.next().into(); + datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap(); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_groups_prevent_project_deletion() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_groups_prevent_project_deletion", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, mut project) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // If we try to delete the project, we'll fail. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "project to be deleted contains an anti affinity group" + ), + "{err:?}" + ); + + // Delete the group, then try to delete the project again. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + // When the group was created, it bumped the rcgen in the project. If we + // have an old view of the project, we expect a "concurrent + // modification" error. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(err.to_string().contains("concurrent modification"), "{err:?}"); + + // If we update rcgen, however, and the group has been deleted, + // we can successfully delete the project. + project.rcgen = project.rcgen.next().into(); + datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap(); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_names_are_unique_in_project() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_names_are_unique_in_project"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two projects + let (authz_project1, _) = + create_project(&opctx, &datastore, "my-project-1").await; + let (authz_project2, _) = + create_project(&opctx, &datastore, "my-project-2").await; + + // We can create a group wiht the same name in different projects + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap(); + create_affinity_group(&opctx, &datastore, &authz_project2, "my-group") + .await + .unwrap(); + + // If we try to create a new group with the same name in the same + // project, we'll see an error. + let err = create_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::ObjectAlreadyExists { + type_name, + object_name, + } if *type_name == ResourceType::AffinityGroup && + *object_name == "my-group"), + "Unexpected error: {err:?}" + ); + + // If we delete the group from the project, we can re-use the name. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + create_affinity_group(&opctx, &datastore, &authz_project1, "my-group") + .await + .expect("Should have been able to re-use name after deletion"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_names_are_unique_in_project() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_names_are_unique_in_project", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two projects + let (authz_project1, _) = + create_project(&opctx, &datastore, "my-project-1").await; + let (authz_project2, _) = + create_project(&opctx, &datastore, "my-project-2").await; + + // We can create a group wiht the same name in different projects + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap(); + create_anti_affinity_group( + &opctx, + &datastore, + &authz_project2, + "my-group", + ) + .await + .unwrap(); + + // If we try to create a new group with the same name in the same + // project, we'll see an error. + let err = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::ObjectAlreadyExists { + type_name, + object_name, + } if *type_name == ResourceType::AntiAffinityGroup && + *object_name == "my-group"), + "Unexpected error: {err:?}" + ); + + // If we delete the group from the project, we can re-use the name. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .expect("Should have been able to re-use name after deletion"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_add_list_remove() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_add_list_remove"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance as a member to the group + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // We should now be able to list the new member + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_add_list_remove() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_list_remove", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance as a member to the group + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_add_remove_instance_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_membership_add_remove_instance_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance with a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + set_instance_state_running(&datastore, instance.id()).await; + + // Cannot add the instance to the group while it's running. + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Instance cannot be added to affinity group in state" + ), + "{err:?}" + ); + + // If we stop the instance, we can add it to the group. + set_instance_state_stopped(&datastore, instance.id()).await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Now we can set the instance state to "running" once more. + set_instance_state_running(&datastore, instance.id()).await; + + // We should now be able to list the new member + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list -- even + // though it's running! + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_add_remove_instance_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_remove_instance_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance with a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + set_instance_state_running(&datastore, instance.id()).await; + + // Cannot add the instance to the group while it's running. + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to anti-affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Instance cannot be added to anti-affinity group in state" + ), + "{err:?}" + ); + + // If we stop the instance, we can add it to the group. + set_instance_state_stopped(&datastore, instance.id()).await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Now we can set the instance state to "running" once more. + set_instance_state_running(&datastore, instance.id()).await; + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list -- even + // though it's running! + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_delete_group_deletes_members() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_delete_group_deletes_members"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the group + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + // Confirm that no instance members exist + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_delete_group_deletes_members() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_delete_group_deletes_members", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the group + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_delete_instance_deletes_membership() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_delete_instance_deletes_membership", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the instance + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_delete_instance_deletes_membership() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_delete_instance_deletes_membership", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the instance + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_for_deleted_objects() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_membership_for_deleted_objects", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + + struct TestArgs { + // Does the group exist? + group: bool, + // Does the instance exist? + instance: bool, + } + + let args = [ + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, + ]; + + for arg in args { + // Create an affinity group, and maybe delete it. + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.group { + datastore + .affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + + // Create an instance, and maybe delete it. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + if !arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + + // Try to add the instance to the group. + // + // Expect to see specific errors, depending on whether or not the + // group/instance exist. + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Do the same thing, but for group membership removal. + let err = datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Cleanup, if we actually created anything. + if arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + if arg.group { + datastore + .affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + } + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_for_deleted_objects() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_for_deleted_objects", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + + struct TestArgs { + // Does the group exist? + group: bool, + // Does the instance exist? + instance: bool, + } + + let args = [ + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, + ]; + + for arg in args { + // Create an anti-affinity group, and maybe delete it. + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.group { + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + + // Create an instance, and maybe delete it. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + if !arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + + // Try to add the instance to the group. + // + // Expect to see specific errors, depending on whether or not the + // group/instance exist. + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AntiAffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Do the same thing, but for group membership removal. + let err = datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AntiAffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Cleanup, if we actually created anything. + if arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + if arg.group { + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + } + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_idempotency() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_idempotency"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Create an instance + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the group + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Add the instance to the group again + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectAlreadyExists { + type_name: ResourceType::AffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + // We should still only observe a single member in the group. + // + // Two calls to "affinity_group_member_add" should be the same + // as a single call. + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + + // We should be able to delete the membership idempotently. + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let err = datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectNotFound { + type_name: ResourceType::AffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_idempotency() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_group_membership_idempotency"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Create an instance + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the group + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Add the instance to the group again + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectAlreadyExists { + type_name: ResourceType::AntiAffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + // We should still only observe a single member in the group. + // + // Two calls to "anti_affinity_group_member_add" should be the same + // as a single call. + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + + // We should be able to delete the membership idempotently. + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let err = datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectNotFound { + type_name: ResourceType::AntiAffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 65d42fbe7c2..9e4bec81d05 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -1387,6 +1387,13 @@ impl DataStore { })?; let instance_id = InstanceUuid::from_untyped_uuid(authz_instance.id()); + self.instance_affinity_group_memberships_delete(opctx, instance_id) + .await?; + self.instance_anti_affinity_group_memberships_delete( + opctx, + instance_id, + ) + .await?; self.instance_ssh_keys_delete(opctx, instance_id).await?; self.instance_mark_migrations_deleted(opctx, instance_id).await?; diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b2d9f8f2471..5c241578a50 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -49,6 +49,7 @@ use std::num::NonZeroU32; use std::sync::Arc; mod address_lot; +mod affinity; mod allow_list; mod auth; mod bfd; diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index 58b7b315c1c..367e094b2e5 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -225,6 +225,8 @@ impl DataStore { generate_fn_to_ensure_none_in_project!(project_image, name, String); generate_fn_to_ensure_none_in_project!(snapshot, name, String); generate_fn_to_ensure_none_in_project!(vpc, name, String); + generate_fn_to_ensure_none_in_project!(affinity_group, name, String); + generate_fn_to_ensure_none_in_project!(anti_affinity_group, name, String); /// Delete a project pub async fn project_delete( @@ -242,6 +244,9 @@ impl DataStore { self.ensure_no_project_images_in_project(opctx, authz_project).await?; self.ensure_no_snapshots_in_project(opctx, authz_project).await?; self.ensure_no_vpcs_in_project(opctx, authz_project).await?; + self.ensure_no_affinity_groups_in_project(opctx, authz_project).await?; + self.ensure_no_anti_affinity_groups_in_project(opctx, authz_project) + .await?; use db::schema::project::dsl; diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index c629dbfc425..d4591024e7a 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -175,6 +175,16 @@ impl<'a> LookupPath<'a> { Instance::PrimaryKey(Root { lookup_root: self }, id) } + /// Select a resource of type AffinityGroup, identified by its id + pub fn affinity_group_id(self, id: Uuid) -> AffinityGroup<'a> { + AffinityGroup::PrimaryKey(Root { lookup_root: self }, id) + } + + /// Select a resource of type AntiAffinityGroup, identified by its id + pub fn anti_affinity_group_id(self, id: Uuid) -> AntiAffinityGroup<'a> { + AntiAffinityGroup::PrimaryKey(Root { lookup_root: self }, id) + } + /// Select a resource of type IpPool, identified by its name pub fn ip_pool_name<'b, 'c>(self, name: &'b Name) -> IpPool<'c> where @@ -645,7 +655,7 @@ lookup_resource! { lookup_resource! { name = "Project", ancestors = [ "Silo" ], - children = [ "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], + children = [ "AffinityGroup", "AntiAffinityGroup", "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], lookup_by_name = true, soft_deletes = true, primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] @@ -696,6 +706,24 @@ lookup_resource! { primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] } +lookup_resource! { + name = "AffinityGroup", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + +lookup_resource! { + name = "AntiAffinityGroup", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + lookup_resource! { name = "InstanceNetworkInterface", ancestors = [ "Silo", "Project", "Instance" ], From 772e64fa9c8774dbc90a037ead09b59107cf8b56 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:29:43 -0800 Subject: [PATCH 04/84] [nexus] Consider Affinity/Anti-Affinity Groups during instance placement --- nexus/db-queries/src/db/datastore/sled.rs | 1328 ++++++++++++++++- nexus/db-queries/src/db/queries/affinity.rs | 608 ++++++++ nexus/db-queries/src/db/queries/mod.rs | 1 + .../src/policy_test/resource_builder.rs | 2 + nexus/db-queries/src/policy_test/resources.rs | 17 + nexus/db-queries/tests/output/authz-roles.out | 84 ++ .../output/lookup_affinity_sleds_query.sql | 30 + .../lookup_anti_affinity_sleds_query.sql | 32 + 8 files changed, 2079 insertions(+), 23 deletions(-) create mode 100644 nexus/db-queries/src/db/queries/affinity.rs create mode 100644 nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql create mode 100644 nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index ee4e6c31cf7..b402cade0fd 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -13,6 +13,7 @@ use crate::db::datastore::ValidateTransition; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::to_db_sled_policy; +use crate::db::model::AffinityPolicy; use crate::db::model::Sled; use crate::db::model::SledResource; use crate::db::model::SledState; @@ -20,6 +21,8 @@ use crate::db::model::SledUpdate; use crate::db::pagination::paginated; use crate::db::pagination::Paginator; use crate::db::pool::DbConnection; +use crate::db::queries::affinity::lookup_affinity_sleds_query; +use crate::db::queries::affinity::lookup_anti_affinity_sleds_query; use crate::db::update_and_check::{UpdateAndCheck, UpdateStatus}; use crate::db::TransactionError; use crate::transaction_retry::OptionalError; @@ -43,11 +46,34 @@ use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; +use std::collections::HashSet; use std::fmt; use strum::IntoEnumIterator; use thiserror::Error; use uuid::Uuid; +#[derive(Debug, thiserror::Error)] +enum SledReservationError { + #[error("No reservation could be found")] + NotFound, + #[error("More than one sled was found with 'affinity = required'")] + TooManyAffinityConstraints, + #[error("A sled that is required is also considered banned by anti-affinity rules")] + ConflictingAntiAndAffinityConstraints, + #[error("A sled required by an affinity group is not a valid target for other reasons")] + RequiredAffinitySledNotValid, +} + +#[derive(Debug, thiserror::Error)] +enum SledReservationTransactionError { + #[error(transparent)] + Connection(#[from] Error), + #[error(transparent)] + Diesel(#[from] diesel::result::Error), + #[error(transparent)] + Reservation(#[from] SledReservationError), +} + impl DataStore { /// Stores a new sled in the database. /// @@ -192,13 +218,43 @@ impl DataStore { resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> CreateResult { - #[derive(Debug)] - enum SledReservationError { - NotFound, - } + self.sled_reservation_create_inner( + opctx, + instance_id, + propolis_id, + resources, + constraints, + ) + .await + .map_err(|e| match e { + SledReservationTransactionError::Connection(e) => e, + SledReservationTransactionError::Diesel(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + SledReservationTransactionError::Reservation(e) => match e { + SledReservationError::NotFound + | SledReservationError::TooManyAffinityConstraints + | SledReservationError::ConflictingAntiAndAffinityConstraints + | SledReservationError::RequiredAffinitySledNotValid => { + return external::Error::insufficient_capacity( + "No sleds can fit the requested instance", + "No sled targets found that had enough \ + capacity to fit the requested instance.", + ); + } + }, + }) + } + async fn sled_reservation_create_inner( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, + resources: db::model::Resources, + constraints: db::model::SledReservationConstraints, + ) -> Result { let err = OptionalError::new(); - let conn = self.pool_connection_authorized(opctx).await?; self.transaction_retry_wrapper("sled_reservation_create") @@ -291,30 +347,197 @@ impl DataStore { define_sql_function!(fn random() -> diesel::sql_types::Float); - // We only actually care about one target here, so this - // query should have a `.limit(1)` attached. We fetch all - // sled targets to leave additional debugging information in - // the logs, for now. + // Fetch all viable sled targets let sled_targets = sled_targets .order(random()) .get_results_async::(&conn) .await?; + info!( opctx.log, - "found {} available sled targets", sled_targets.len(); + "found {} available sled targets before considering affinity", sled_targets.len(); "sled_ids" => ?sled_targets, ); - if sled_targets.is_empty() { - return Err(err.bail(SledReservationError::NotFound)); + let anti_affinity_sleds = lookup_anti_affinity_sleds_query( + instance_id, + ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; + + let affinity_sleds = lookup_affinity_sleds_query( + instance_id, + ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; + + // We use the following logic to calculate a desirable sled, + // given a possible set of "targets", and the information + // from affinity groups. + // + // # Rules vs Preferences + // + // Due to the flavors "affinity policy", it's possible to bucket affinity + // choices into two categories: "rules" and "preferences". "rules" are affinity + // dispositions for or against sled placement that must be followed, and + // "preferences" are affinity dispositions that should be followed for sled + // selection, in order of "most preferential" to "least preferential". + // + // As example of a "rule" is "an anti-affinity group exists, containing a + // target sled, with affinity_policy = 'fail'". + // + // An example of a "preference" is "an anti-affinity group exists, containing a + // target sled, but the policy is 'allow'. We don't want to use it as a target, + // but we will if there are no other choices." + // + // We apply rules before preferences to ensure they are always respected. + // Furthermore, the evaluation of preferences is a target-seeking operation, + // which identifies the distinct sets of targets, and searches them in + // decreasing preference order. + // + // # Logic + // + // ## Background: Notation + // + // We use the following symbols for sets below: + // - ∩: Intersection of two sets (A ∩ B is "everything that exists in A and + // also exists in B"). + // - \: difference of two sets (A \ B is "everything that exists in A that does + // not exist in B). + // + // We also use the following notation for brevity: + // - AA,P=Fail: All sleds sharing an anti-affinity instance within a group with + // policy = 'fail'. + // - AA,P=Allow: Same as above, but with policy = 'allow'. + // - A,P=Fail: All sleds sharing an affinity instance within a group with + // policy = 'fail'. + // - A,P=Allow: Same as above, but with policy = 'allow'. + // + // ## Affinity: Apply Rules + // + // - Targets := All viable sleds for instance placement + // - Banned := AA,P=Fail + // - Required := A,P=Fail + // - if Required.len() > 1: Fail (too many constraints). + // - if Required.len() == 1... + // - ... if the entry exists in the "Banned" set: Fail + // (contradicting constraints 'Banned' + 'Required') + // - ... if the entry does not exist in "Targets": Fail + // ('Required' constraint not satisfiable) + // - ... if the entry does not exist in "Banned": Use it. + // + // If we have not yet picked a target, we can filter the + // set of targets to ignore "banned" sleds, and then apply + // preferences. + // + // - Targets := Targets \ Banned + // + // ## Affinity: Apply Preferences + // + // - Preferred := Targets ∩ A,P=Allow + // - Unpreferred := Targets ∩ AA,P=Allow + // - Neither := Preferred ∩ Unpreferred + // - Preferred := Preferred \ Neither + // - Unpreferred := Unpreferred \ Neither + // - If Preferred isn't empty, pick a target from it. + // - Targets := Targets \ Unpreferred + // - If Targets isn't empty, pick a target from it. + // - If Unpreferred isn't empty, pick a target from it. + // - Fail, no targets are available. + + let mut targets: HashSet<_> = sled_targets.into_iter().collect(); + + let banned = anti_affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Fail { + Some(*id) + } else { + None + } + }).collect::>(); + let required = affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Fail { + Some(*id) + } else { + None + } + }).collect::>(); + + if !banned.is_empty() { + info!( + opctx.log, + "affinity policy prohibits placement on {} sleds", banned.len(); + "banned" => ?banned, + ); + } + if !required.is_empty() { + info!( + opctx.log, + "affinity policy requires placement on {} sleds", required.len(); + "required" => ?required, + ); } + let sled_target = if required.len() > 1 { + return Err(err.bail(SledReservationError::TooManyAffinityConstraints)); + } else if let Some(required_id) = required.iter().next() { + // If we have a "required" sled, it must be chosen. + + if banned.contains(&required_id) { + return Err(err.bail(SledReservationError::ConflictingAntiAndAffinityConstraints)); + } + if !targets.contains(&required_id) { + return Err(err.bail(SledReservationError::RequiredAffinitySledNotValid)); + } + *required_id + } else { + // We have no "required" sleds, but might have preferences. + + targets = targets.difference(&banned).cloned().collect(); + + let mut preferred = affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Allow { + Some(*id) + } else { + None + } + }).collect::>(); + let mut unpreferred = anti_affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Allow { + Some(*id) + } else { + None + } + }).collect::>(); + + // Only consider "preferred" sleds that are viable targets + preferred = targets.intersection(&preferred).cloned().collect(); + // Only consider "unpreferred" sleds that are viable targets + unpreferred = targets.intersection(&unpreferred).cloned().collect(); + + // If a target is both preferred and unpreferred, it is removed + // from both sets. + let both = preferred.intersection(&unpreferred).cloned().collect(); + preferred = preferred.difference(&both).cloned().collect(); + unpreferred = unpreferred.difference(&both).cloned().collect(); + + if let Some(target) = preferred.iter().next() { + *target + } else { + targets = targets.difference(&unpreferred).cloned().collect(); + if let Some(target) = targets.iter().next() { + *target + } else { + if let Some(target) = unpreferred.iter().next() { + *target + } else { + return Err(err.bail(SledReservationError::NotFound)); + } + } + } + }; + // Create a SledResource record, associate it with the target // sled. let resource = SledResource::new_for_vmm( propolis_id, instance_id, - SledUuid::from_untyped_uuid(sled_targets[0]), + SledUuid::from_untyped_uuid(sled_target), resources, ); @@ -328,17 +551,9 @@ impl DataStore { .await .map_err(|e| { if let Some(err) = err.take() { - match err { - SledReservationError::NotFound => { - return external::Error::insufficient_capacity( - "No sleds can fit the requested instance", - "No sled targets found that had enough \ - capacity to fit the requested instance.", - ); - } - } + return SledReservationTransactionError::Reservation(err); } - public_error_from_diesel(e, ErrorHandler::Server) + SledReservationTransactionError::Diesel(e) }) } @@ -849,7 +1064,10 @@ pub(in crate::db::datastore) mod test { }; use crate::db::lookup::LookupPath; use crate::db::model::to_db_typed_uuid; + use crate::db::model::AffinityGroup; + use crate::db::model::AntiAffinityGroup; use crate::db::model::ByteCount; + use crate::db::model::Project; use crate::db::model::SqlU32; use crate::db::pub_test_utils::TestDatabase; use anyhow::{Context, Result}; @@ -859,13 +1077,16 @@ pub(in crate::db::datastore) mod test { use nexus_db_model::PhysicalDiskKind; use nexus_db_model::PhysicalDiskPolicy; use nexus_db_model::PhysicalDiskState; + use nexus_types::external_api::params; use nexus_types::identity::Asset; + use nexus_types::identity::Resource; use omicron_common::api::external; use omicron_test_utils::dev; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use predicates::{prelude::*, BoxPredicate}; + use std::collections::HashMap; use std::net::{Ipv6Addr, SocketAddrV6}; fn rack_id() -> Uuid { @@ -1159,6 +1380,1067 @@ pub(in crate::db::datastore) mod test { logctx.cleanup_successful(); } + // Utilities to help with Affinity Testing + + // Create a resource request that will probably fit on a sled. + fn small_resource_request() -> db::model::Resources { + db::model::Resources::new( + 1, + // Just require the bare non-zero amount of RAM. + ByteCount::try_from(1024).unwrap(), + ByteCount::try_from(1024).unwrap(), + ) + } + + // Create a resource request that will entirely fill a sled. + fn large_resource_request() -> db::model::Resources { + let sled_resources = sled_system_hardware_for_test(); + let threads = sled_resources.usable_hardware_threads; + let rss_ram = sled_resources.usable_physical_ram; + let reservoir_ram = sled_resources.reservoir_size; + db::model::Resources::new(threads, rss_ram, reservoir_ram) + } + + async fn create_project( + opctx: &OpContext, + datastore: &DataStore, + ) -> (authz::Project, db::model::Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + + // Create a project + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: external::IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: "desc".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() + } + + async fn create_anti_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &'static str, + policy: external::AffinityPolicy, + ) -> AntiAffinityGroup { + datastore + .anti_affinity_group_create( + &opctx, + &authz_project, + AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap() + } + + // This short-circuits some of the logic and checks we normally have when + // creating affinity groups, but makes testing easier. + async fn add_instance_to_anti_affinity_group( + datastore: &DataStore, + group_id: Uuid, + instance_id: Uuid, + ) { + use db::model::AntiAffinityGroupInstanceMembership; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use omicron_uuid_kinds::AntiAffinityGroupUuid; + + diesel::insert_into( + membership_dsl::anti_affinity_group_instance_membership, + ) + .values(AntiAffinityGroupInstanceMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(group_id), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .on_conflict((membership_dsl::group_id, membership_dsl::instance_id)) + .do_nothing() + .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap(); + } + + async fn create_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &'static str, + policy: external::AffinityPolicy, + ) -> AffinityGroup { + datastore + .affinity_group_create( + &opctx, + &authz_project, + AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap() + } + + // This short-circuits some of the logic and checks we normally have when + // creating affinity groups, but makes testing easier. + async fn add_instance_to_affinity_group( + datastore: &DataStore, + group_id: Uuid, + instance_id: Uuid, + ) { + use db::model::AffinityGroupInstanceMembership; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use omicron_uuid_kinds::AffinityGroupUuid; + + diesel::insert_into(membership_dsl::affinity_group_instance_membership) + .values(AffinityGroupInstanceMembership::new( + AffinityGroupUuid::from_untyped_uuid(group_id), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .on_conflict(( + membership_dsl::group_id, + membership_dsl::instance_id, + )) + .do_nothing() + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { + let mut sleds = vec![]; + for _ in 0..count { + let (sled, _) = + datastore.sled_upsert(test_new_sled_update()).await.unwrap(); + sleds.push(sled); + } + sleds + } + + type GroupName = &'static str; + + #[derive(Copy, Clone, Debug)] + enum Affinity { + Positive, + Negative, + } + + struct Group { + affinity: Affinity, + name: GroupName, + policy: external::AffinityPolicy, + } + + impl Group { + async fn create( + &self, + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + ) -> Uuid { + match self.affinity { + Affinity::Positive => create_affinity_group( + &opctx, + &datastore, + &authz_project, + self.name, + self.policy, + ) + .await + .id(), + Affinity::Negative => create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + self.name, + self.policy, + ) + .await + .id(), + } + } + } + + struct AllGroups { + id_by_name: HashMap<&'static str, (Affinity, Uuid)>, + } + + impl AllGroups { + async fn create( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + groups: &[Group], + ) -> AllGroups { + let mut id_by_name = HashMap::new(); + + for group in groups { + id_by_name.insert( + group.name, + ( + group.affinity, + group.create(&opctx, &datastore, &authz_project).await, + ), + ); + } + + Self { id_by_name } + } + } + + struct Instance { + id: InstanceUuid, + groups: Vec, + force_onto_sled: Option, + resources: db::model::Resources, + } + + impl Instance { + fn new() -> Self { + Self { + id: InstanceUuid::new_v4(), + groups: vec![], + force_onto_sled: None, + resources: small_resource_request(), + } + } + + fn use_many_resources(mut self) -> Self { + self.resources = large_resource_request(); + self + } + + // Adds this instance to a group. Can be called multiple times. + fn group(mut self, group: GroupName) -> Self { + self.groups.push(group); + self + } + + // Force this instance to be placed on a specific sled + fn sled(mut self, sled: Uuid) -> Self { + self.force_onto_sled = Some(SledUuid::from_untyped_uuid(sled)); + self + } + + async fn add_to_groups_and_reserve( + &self, + opctx: &OpContext, + datastore: &DataStore, + all_groups: &AllGroups, + ) -> Result + { + self.add_to_groups(&datastore, &all_groups).await; + create_instance_reservation(&datastore, &opctx, self).await + } + + async fn add_to_groups( + &self, + datastore: &DataStore, + all_groups: &AllGroups, + ) { + for our_group_name in &self.groups { + let (affinity, group_id) = all_groups + .id_by_name + .get(our_group_name) + .expect("Group not found: {our_group_name}"); + match *affinity { + Affinity::Positive => { + add_instance_to_affinity_group( + &datastore, + *group_id, + self.id.into_untyped_uuid(), + ) + .await + } + Affinity::Negative => { + add_instance_to_anti_affinity_group( + &datastore, + *group_id, + self.id.into_untyped_uuid(), + ) + .await + } + } + } + } + } + + async fn create_instance_reservation( + db: &DataStore, + opctx: &OpContext, + instance: &Instance, + ) -> Result { + // Pick a specific sled, if requested + let constraints = db::model::SledReservationConstraintBuilder::new(); + let constraints = if let Some(sled_target) = instance.force_onto_sled { + constraints.must_select_from(&[sled_target.into_untyped_uuid()]) + } else { + constraints + }; + + // We're accessing the inner implementation to get a more detailed + // view of the error code. + let result = db + .sled_reservation_create_inner( + &opctx, + instance.id, + PropolisUuid::new_v4(), + instance.resources.clone(), + constraints.build(), + ) + .await?; + + if let Some(sled_target) = instance.force_onto_sled { + assert_eq!(SledUuid::from(result.sled_id), sled_target); + } + + Ok(result) + } + + // Anti-Affinity, Policy = Fail + // No sleds available => Insufficient Capacity Error + #[tokio::test] + async fn anti_affinity_policy_fail_no_capacity() { + let logctx = + dev::test_setup_log("anti_affinity_policy_fail_no_capacity"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new().group("anti-affinity").sled(sleds[0].id()), + Instance::new().group("anti-affinity").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + + let SledReservationTransactionError::Reservation( + SledReservationError::NotFound, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Fail + // We should reliably pick a sled not occupied by another instance + #[tokio::test] + async fn anti_affinity_policy_fail() { + let logctx = dev::test_setup_log("anti_affinity_policy_fail"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new().group("anti-affinity").sled(sleds[0].id()), + Instance::new().group("anti-affinity").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[2].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Allow + // + // Create two sleds, put instances on each belonging to an anti-affinity group. + // We can continue to add instances belonging to that anti-affinity group, because + // it has "AffinityPolicy::Allow". + #[tokio::test] + async fn anti_affinity_policy_allow() { + let logctx = dev::test_setup_log("anti_affinity_policy_allow"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new().group("anti-affinity").sled(sleds[0].id()), + Instance::new().group("anti-affinity").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + assert!( + [sleds[0].id(), sleds[1].id()] + .contains(resource.sled_id.as_untyped_uuid()), + "Should have been provisioned to one of the two viable sleds" + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Fail + // + // Placement of instances with positive affinity will share a sled. + #[tokio::test] + async fn affinity_policy_fail() { + let logctx = dev::test_setup_log("affinity_policy_fail"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new().group("affinity").sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have placed instance"); + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[0].id()); + + let another_test_instance = Instance::new().group("affinity"); + let resource = another_test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have placed instance (again)"); + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[0].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Fail, with too many constraints. + #[tokio::test] + async fn affinity_policy_fail_too_many_constraints() { + let logctx = + dev::test_setup_log("affinity_policy_fail_too_many_constraints"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // We are constrained so that an instance belonging to both groups must be + // placed on both sled 0 and sled 1. This cannot be satisfied, and it returns + // an error. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity2").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("affinity1").group("affinity2"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + + let SledReservationTransactionError::Reservation( + SledReservationError::TooManyAffinityConstraints, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Fail forces "no space" early + // + // Placement of instances with positive affinity with "policy = Fail" + // will always be forced to share a sled, even if there are other options. + #[tokio::test] + async fn affinity_policy_fail_no_capacity() { + let logctx = dev::test_setup_log("affinity_policy_fail_no_capacity"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new() + .use_many_resources() + .group("affinity") + .sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().use_many_resources().group("affinity"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + let SledReservationTransactionError::Reservation( + SledReservationError::RequiredAffinitySledNotValid, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Allow lets us use other sleds + // + // This is similar to "affinity_policy_fail_no_capacity", but by + // using "Policy = Allow" instead of "Policy = Fail", we are able to pick + // a different sled for the reservation. + #[tokio::test] + async fn affinity_policy_allow_picks_different_sled() { + let logctx = + dev::test_setup_log("affinity_policy_allow_picks_different_sled"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new() + .use_many_resources() + .group("affinity") + .sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().use_many_resources().group("affinity"); + let reservation = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have made reservation"); + assert_eq!(reservation.sled_id.into_untyped_uuid(), sleds[1].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Fail + Affinity, Policy = Fail + // + // These constraints are contradictory - we're asking the allocator + // to colocate and NOT colocate our new instance with existing ones, which + // should not be satisfiable. + #[tokio::test] + async fn affinity_and_anti_affinity_policy_fail() { + let logctx = + dev::test_setup_log("affinity_and_anti_affinity_policy_fail"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new() + .group("anti-affinity") + .group("affinity") + .sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("anti-affinity").group("affinity"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Contradictory constraints should not be satisfiable"); + let SledReservationTransactionError::Reservation( + SledReservationError::ConflictingAntiAndAffinityConstraints, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Allow + Affinity, Policy = Allow + // + // These constraints are contradictory, but since they encode + // "preferences" rather than "rules", they cancel out. + #[tokio::test] + async fn affinity_and_anti_affinity_policy_allow() { + let logctx = + dev::test_setup_log("affinity_and_anti_affinity_policy_allow"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new() + .group("anti-affinity") + .group("affinity") + .sled(sleds[0].id()), + Instance::new() + .group("anti-affinity") + .group("affinity") + .sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("anti-affinity").group("affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + assert!( + [sleds[0].id(), sleds[1].id()] + .contains(resource.sled_id.as_untyped_uuid()), + "Should have been provisioned to one of the two viable sleds" + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_multi_group() { + let logctx = dev::test_setup_log("anti_affinity_multi_group"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "strict-anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Sleds 0 and 1 contain the "strict-anti-affinity" group instances, + // and won't be used. + // + // Sled 3 has an "anti-affinity" group instance, and also won't be used. + // + // This only leaves sled 2. + let instances = [ + Instance::new() + .group("strict-anti-affinity") + .group("affinity") + .sled(sleds[0].id()), + Instance::new().group("strict-anti-affinity").sled(sleds[1].id()), + Instance::new().group("anti-affinity").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new() + .group("strict-anti-affinity") + .group("anti-affinity") + .group("affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[2].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_multi_group() { + let logctx = dev::test_setup_log("affinity_multi_group"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Sled 0 contains an affinity group, but it's large enough to make it + // non-viable for future allocations. + // + // Sled 1 contains an affinity and anti-affinity group, so they cancel out. + // This gives it "no priority". + // + // Sled 2 contains nothing. It's a viable target, neither preferred nor + // unpreferred. + // + // Sled 3 contains an affinity group, which is prioritized. + let instances = [ + Instance::new() + .group("affinity") + .use_many_resources() + .sled(sleds[0].id()), + Instance::new() + .group("affinity") + .group("anti-affinity") + .sled(sleds[1].id()), + Instance::new().group("affinity").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("affinity").group("anti-affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[3].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_ignored_from_other_groups() { + let logctx = dev::test_setup_log("affinity_ignored_from_other_groups"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Only "sleds[1]" has space. We ignore the affinity policy because + // our new instance won't belong to either group. + let instances = [ + Instance::new() + .group("affinity1") + .use_many_resources() + .sled(sleds[0].id()), + Instance::new() + .group("affinity2") + .use_many_resources() + .sled(sleds[2].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new(); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[1].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_ignored_from_other_groups() { + let logctx = + dev::test_setup_log("anti_affinity_ignored_from_other_groups"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity1", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Negative, + name: "anti-affinity2", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Only "sleds[2]" has space, even though it also contains an anti-affinity group. + // However, if we don't belong to this group, we won't care. + + let instances = [ + Instance::new().group("anti-affinity1").sled(sleds[0].id()), + Instance::new().use_many_resources().sled(sleds[1].id()), + Instance::new().group("anti-affinity2").sled(sleds[2].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity1"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[2].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + async fn lookup_physical_disk( datastore: &DataStore, id: PhysicalDiskUuid, diff --git a/nexus/db-queries/src/db/queries/affinity.rs b/nexus/db-queries/src/db/queries/affinity.rs new file mode 100644 index 00000000000..6da857669da --- /dev/null +++ b/nexus/db-queries/src/db/queries/affinity.rs @@ -0,0 +1,608 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of queries for affinity groups + +use crate::db::model::AffinityPolicyEnum; +use crate::db::raw_query_builder::{QueryBuilder, TypedSqlQuery}; +use diesel::sql_types; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; + +/// For an instance, look up all anti-affinity groups, and return +/// a list of sleds with other instances in that anti-affinity group +/// already reserved. +pub fn lookup_anti_affinity_sleds_query( + instance_id: InstanceUuid, +) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { + QueryBuilder::new().sql( + "WITH our_groups AS ( + SELECT group_id + FROM anti_affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_instances AS ( + SELECT anti_affinity_group_instance_membership.group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_groups + ON anti_affinity_group_instance_membership.group_id = our_groups.group_id + WHERE instance_id != ").param().sql(" + ), + other_instances_by_policy AS ( + SELECT policy,instance_id + FROM other_instances + JOIN anti_affinity_group + ON + anti_affinity_group.id = other_instances.group_id AND + anti_affinity_group.failure_domain = 'sled' + WHERE anti_affinity_group.time_deleted IS NULL + ) + SELECT DISTINCT policy,sled_id + FROM other_instances_by_policy + JOIN sled_resource + ON + sled_resource.instance_id = other_instances_by_policy.instance_id AND + sled_resource.kind = 'instance'") + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .query() +} + +/// For an instance, look up all affinity groups, and return a list of sleds +/// with other instances in that affinity group already reserved. +pub fn lookup_affinity_sleds_query( + instance_id: InstanceUuid, +) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { + QueryBuilder::new() + .sql( + "WITH our_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ", + ) + .param() + .sql( + " + ), + other_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_groups + ON affinity_group_instance_membership.group_id = our_groups.group_id + WHERE instance_id != ", + ) + .param() + .sql( + " + ), + other_instances_by_policy AS ( + SELECT policy,instance_id + FROM other_instances + JOIN affinity_group + ON + affinity_group.id = other_instances.group_id AND + affinity_group.failure_domain = 'sled' + WHERE affinity_group.time_deleted IS NULL + ) + SELECT DISTINCT policy,sled_id + FROM other_instances_by_policy + JOIN sled_resource + ON + sled_resource.instance_id = other_instances_by_policy.instance_id AND + sled_resource.kind = 'instance'", + ) + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .query() +} + +#[cfg(test)] +mod test { + use super::*; + use crate::db::explain::ExplainableAsync; + use crate::db::model; + use crate::db::pub_test_utils::TestDatabase; + use crate::db::raw_query_builder::expectorate_query_contents; + + use anyhow::Context; + use async_bb8_diesel::AsyncRunQueryDsl; + use nexus_types::external_api::params; + use nexus_types::identity::Resource; + use omicron_common::api::external; + use omicron_test_utils::dev; + use omicron_uuid_kinds::AffinityGroupUuid; + use omicron_uuid_kinds::AntiAffinityGroupUuid; + use omicron_uuid_kinds::PropolisUuid; + use omicron_uuid_kinds::SledUuid; + use uuid::Uuid; + + #[tokio::test] + async fn expectorate_lookup_anti_affinity_sleds_query() { + let id = InstanceUuid::nil(); + + let query = lookup_anti_affinity_sleds_query(id); + expectorate_query_contents( + &query, + "tests/output/lookup_anti_affinity_sleds_query.sql", + ) + .await; + } + + #[tokio::test] + async fn expectorate_lookup_affinity_sleds_query() { + let id = InstanceUuid::nil(); + + let query = lookup_affinity_sleds_query(id); + expectorate_query_contents( + &query, + "tests/output/lookup_affinity_sleds_query.sql", + ) + .await; + } + + #[tokio::test] + async fn explain_lookup_anti_affinity_sleds_query() { + let logctx = + dev::test_setup_log("explain_lookup_anti_affinity_sleds_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let id = InstanceUuid::nil(); + let query = lookup_anti_affinity_sleds_query(id); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn explain_lookup_affinity_sleds_query() { + let logctx = dev::test_setup_log("explain_lookup_affinity_sleds_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let id = InstanceUuid::nil(); + let query = lookup_affinity_sleds_query(id); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.terminate().await; + logctx.cleanup_successful(); + } + + async fn make_affinity_group( + project_id: Uuid, + name: &'static str, + policy: external::AffinityPolicy, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result { + let group = model::AffinityGroup::new( + project_id, + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ); + use crate::db::schema::affinity_group::dsl; + diesel::insert_into(dsl::affinity_group) + .values(group.clone()) + .execute_async(conn) + .await + .context("Cannot create affinity group")?; + Ok(group) + } + + async fn make_anti_affinity_group( + project_id: Uuid, + name: &'static str, + policy: external::AffinityPolicy, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result { + let group = model::AntiAffinityGroup::new( + project_id, + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ); + use crate::db::schema::anti_affinity_group::dsl; + diesel::insert_into(dsl::anti_affinity_group) + .values(group.clone()) + .execute_async(conn) + .await + .context("Cannot create anti affinity group")?; + Ok(group) + } + + async fn make_affinity_group_instance_membership( + group_id: AffinityGroupUuid, + instance_id: InstanceUuid, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result<()> { + // Let's claim an instance belongs to that group + let membership = + model::AffinityGroupInstanceMembership::new(group_id, instance_id); + use crate::db::schema::affinity_group_instance_membership::dsl; + diesel::insert_into(dsl::affinity_group_instance_membership) + .values(membership) + .execute_async(conn) + .await + .context("Cannot create affinity group instance membership")?; + Ok(()) + } + + async fn make_anti_affinity_group_instance_membership( + group_id: AntiAffinityGroupUuid, + instance_id: InstanceUuid, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result<()> { + // Let's claim an instance belongs to that group + let membership = model::AntiAffinityGroupInstanceMembership::new( + group_id, + instance_id, + ); + use crate::db::schema::anti_affinity_group_instance_membership::dsl; + diesel::insert_into(dsl::anti_affinity_group_instance_membership) + .values(membership) + .execute_async(conn) + .await + .context("Cannot create anti affinity group instance membership")?; + Ok(()) + } + + async fn make_instance_sled_resource( + sled_id: SledUuid, + instance_id: InstanceUuid, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result<()> { + let resource = model::SledResource::new_for_vmm( + PropolisUuid::new_v4(), + instance_id, + sled_id, + model::Resources::new( + 0, + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + ), + ); + use crate::db::schema::sled_resource::dsl; + diesel::insert_into(dsl::sled_resource) + .values(resource) + .execute_async(conn) + .await + .context("Cannot create sled resource")?; + Ok(()) + } + + #[tokio::test] + async fn lookup_affinity_sleds() { + let logctx = dev::test_setup_log("lookup_affinity_sleds"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let our_instance_id = InstanceUuid::new_v4(); + let other_instance_id = InstanceUuid::new_v4(); + + // With no groups and no instances, we should see no other instances + // belonging to our affinity group. + assert_eq!( + lookup_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![], + ); + + let sled_id = SledUuid::new_v4(); + let project_id = Uuid::new_v4(); + + // Make a group, add our instance to it + let group = make_affinity_group( + project_id, + "group-allow", + external::AffinityPolicy::Allow, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + + // Create an instance which belongs to that group + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + make_instance_sled_resource(sled_id, other_instance_id, &conn) + .await + .unwrap(); + + // Now if we look, we'll find the "other sled", on which the "other + // instance" was placed. + assert_eq!( + lookup_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![(model::AffinityPolicy::Allow, sled_id.into_untyped_uuid())], + ); + + // If we look from the perspective of the other instance, + // we "ignore ourselves" for placement, so the set of sleds to consider + // is still empty. + assert_eq!( + lookup_affinity_sleds_query(other_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![] + ); + + // If we make another group (note the policy is different this time!) + // it'll also be reflected in the results. + let group = make_affinity_group( + project_id, + "group-fail", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + // We see the outcome of both groups, with a different policy for each. + let mut results = lookup_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + // Let's add one more group, to see what happens to the query output. + let group = make_affinity_group( + project_id, + "group-fail2", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + let mut results = lookup_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + + // Since we use "SELECT DISTINCT", the results are bounded, and do not + // grow as we keep on finding more sleds that have the same policy. + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn lookup_anti_affinity_sleds() { + let logctx = dev::test_setup_log("lookup_anti_affinity_sleds"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let our_instance_id = InstanceUuid::new_v4(); + let other_instance_id = InstanceUuid::new_v4(); + + // With no groups and no instances, we should see no other instances + // belonging to our anti-affinity group. + assert_eq!( + lookup_anti_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![], + ); + + let sled_id = SledUuid::new_v4(); + let project_id = Uuid::new_v4(); + + // Make a group, add our instance to it + let group = make_anti_affinity_group( + project_id, + "group-allow", + external::AffinityPolicy::Allow, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + + // Create an instance which belongs to that group + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + make_instance_sled_resource(sled_id, other_instance_id, &conn) + .await + .unwrap(); + + // Now if we look, we'll find the "other sled", on which the "other + // instance" was placed. + assert_eq!( + lookup_anti_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![(model::AffinityPolicy::Allow, sled_id.into_untyped_uuid())], + ); + + // If we look from the perspective of the other instance, + // we "ignore ourselves" for placement, so the set of sleds to consider + // is still empty. + assert_eq!( + lookup_anti_affinity_sleds_query(other_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![] + ); + + // If we make another group (note the policy is different this time!) + // it'll also be reflected in the results. + let group = make_anti_affinity_group( + project_id, + "group-fail", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + // We see the outcome of both groups, with a different policy for each. + let mut results = lookup_anti_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + // Let's add one more group, to see what happens to the query output. + let group = make_anti_affinity_group( + project_id, + "group-fail2", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + let mut results = lookup_anti_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + + // Since we use "SELECT DISTINCT", the results are bounded, and do not + // grow as we keep on finding more sleds that have the same policy. + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index 5f34c7cfb3d..f2af3aebc99 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -5,6 +5,7 @@ //! Specialized queries for inserting database records, usually to maintain //! complex invariants that are most accurately expressed in a single query. +pub mod affinity; pub mod disk; pub mod external_ip; pub mod ip_pool; diff --git a/nexus/db-queries/src/policy_test/resource_builder.rs b/nexus/db-queries/src/policy_test/resource_builder.rs index b6d7d97553e..310c11adf3c 100644 --- a/nexus/db-queries/src/policy_test/resource_builder.rs +++ b/nexus/db-queries/src/policy_test/resource_builder.rs @@ -243,6 +243,8 @@ macro_rules! impl_dyn_authorized_resource_for_resource { } impl_dyn_authorized_resource_for_resource!(authz::AddressLot); +impl_dyn_authorized_resource_for_resource!(authz::AffinityGroup); +impl_dyn_authorized_resource_for_resource!(authz::AntiAffinityGroup); impl_dyn_authorized_resource_for_resource!(authz::Blueprint); impl_dyn_authorized_resource_for_resource!(authz::Certificate); impl_dyn_authorized_resource_for_resource!(authz::DeviceAccessToken); diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index 6ee92e167cf..04655410533 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -300,6 +300,21 @@ async fn make_project( LookupType::ByName(vpc1_name.clone()), ); + let affinity_group_name = format!("{}-affinity-group1", project_name); + let affinity_group = authz::AffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(affinity_group_name.clone()), + ); + + let anti_affinity_group_name = + format!("{}-anti-affinity-group1", project_name); + let anti_affinity_group = authz::AntiAffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(anti_affinity_group_name.clone()), + ); + let instance_name = format!("{}-instance1", project_name); let instance = authz::Instance::new( project.clone(), @@ -313,6 +328,8 @@ async fn make_project( Uuid::new_v4(), LookupType::ByName(disk_name.clone()), )); + builder.new_resource(affinity_group.clone()); + builder.new_resource(anti_affinity_group.clone()); builder.new_resource(instance.clone()); builder.new_resource(authz::InstanceNetworkInterface::new( instance, diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 4b24e649ccb..e0d43250d13 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -306,6 +306,34 @@ resource: Disk "silo1-proj1-disk1" silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj1-instance1" USER Q R LC RP M MP CC D @@ -474,6 +502,34 @@ resource: Disk "silo1-proj2-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj2-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj2-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj2-instance1" USER Q R LC RP M MP CC D @@ -810,6 +866,34 @@ resource: Disk "silo2-proj1-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo2-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo2-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo2-proj1-instance1" USER Q R LC RP M MP CC D diff --git a/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql new file mode 100644 index 00000000000..1435a0b8f62 --- /dev/null +++ b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql @@ -0,0 +1,30 @@ +WITH + our_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $1), + other_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_groups ON affinity_group_instance_membership.group_id = our_groups.group_id + WHERE + instance_id != $2 + ), + other_instances_by_policy + AS ( + SELECT + policy, instance_id + FROM + other_instances + JOIN affinity_group ON + affinity_group.id = other_instances.group_id AND affinity_group.failure_domain = 'sled' + WHERE + affinity_group.time_deleted IS NULL + ) +SELECT + DISTINCT policy, sled_id +FROM + other_instances_by_policy + JOIN sled_resource ON + sled_resource.instance_id = other_instances_by_policy.instance_id + AND sled_resource.kind = 'instance' diff --git a/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql new file mode 100644 index 00000000000..d5e7cd66d86 --- /dev/null +++ b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql @@ -0,0 +1,32 @@ +WITH + our_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $1), + other_instances + AS ( + SELECT + anti_affinity_group_instance_membership.group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_groups ON anti_affinity_group_instance_membership.group_id = our_groups.group_id + WHERE + instance_id != $2 + ), + other_instances_by_policy + AS ( + SELECT + policy, instance_id + FROM + other_instances + JOIN anti_affinity_group ON + anti_affinity_group.id = other_instances.group_id + AND anti_affinity_group.failure_domain = 'sled' + WHERE + anti_affinity_group.time_deleted IS NULL + ) +SELECT + DISTINCT policy, sled_id +FROM + other_instances_by_policy + JOIN sled_resource ON + sled_resource.instance_id = other_instances_by_policy.instance_id + AND sled_resource.kind = 'instance' From d8cff32f565fdf6588fd74851aed8f19413de0fb Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:51:18 -0800 Subject: [PATCH 05/84] [nexus] Implement Affinity/Anti-Affinity Groups in external API --- common/src/api/external/http_pagination.rs | 13 + nexus/src/app/affinity.rs | 399 ++++++++ nexus/src/app/mod.rs | 1 + nexus/src/external_api/http_entrypoints.rs | 430 +++++++-- nexus/test-utils/src/resource_helpers.rs | 44 + nexus/tests/integration_tests/affinity.rs | 895 ++++++++++++++++++ nexus/tests/integration_tests/endpoints.rs | 220 +++++ nexus/tests/integration_tests/mod.rs | 1 + nexus/tests/integration_tests/projects.rs | 78 +- nexus/tests/integration_tests/unauthorized.rs | 31 + .../output/uncovered-authz-endpoints.txt | 18 - 11 files changed, 2053 insertions(+), 77 deletions(-) create mode 100644 nexus/src/app/affinity.rs create mode 100644 nexus/tests/integration_tests/affinity.rs diff --git a/common/src/api/external/http_pagination.rs b/common/src/api/external/http_pagination.rs index d4d237b2341..7cfef9a09c0 100644 --- a/common/src/api/external/http_pagination.rs +++ b/common/src/api/external/http_pagination.rs @@ -312,6 +312,19 @@ pub type PaginatedByNameOrId = PaginationParams< pub type PageSelectorByNameOrId = PageSelector, NameOrId>; +pub fn id_pagination<'a, Selector>( + pag_params: &'a DataPageParams, + scan_params: &'a ScanById, +) -> Result, HttpError> +where + Selector: + Clone + Debug + DeserializeOwned + JsonSchema + PartialEq + Serialize, +{ + match scan_params.sort_by { + IdSortMode::IdAscending => Ok(PaginatedBy::Id(pag_params.clone())), + } +} + pub fn name_or_id_pagination<'a, Selector>( pag_params: &'a DataPageParams, scan_params: &'a ScanByNameOrId, diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs new file mode 100644 index 00000000000..894f29416e7 --- /dev/null +++ b/nexus/src/app/affinity.rs @@ -0,0 +1,399 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Affinity groups + +use nexus_db_model::AffinityGroup; +use nexus_db_model::AntiAffinityGroup; +use nexus_db_queries::authz; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup; +use nexus_db_queries::db::lookup::LookupPath; +use nexus_types::external_api::params; +use nexus_types::external_api::views; +use omicron_common::api::external; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::NameOrId; +use omicron_common::api::external::UpdateResult; + +impl super::Nexus { + pub fn affinity_group_lookup<'a>( + &'a self, + opctx: &'a OpContext, + affinity_group_selector: params::AffinityGroupSelector, + ) -> LookupResult> { + match affinity_group_selector { + params::AffinityGroupSelector { + affinity_group: NameOrId::Id(id), + project: None + } => { + let affinity_group = + LookupPath::new(opctx, &self.db_datastore).affinity_group_id(id); + Ok(affinity_group) + } + params::AffinityGroupSelector { + affinity_group: NameOrId::Name(name), + project: Some(project) + } => { + let affinity_group = self + .project_lookup(opctx, params::ProjectSelector { project })? + .affinity_group_name_owned(name.into()); + Ok(affinity_group) + } + params::AffinityGroupSelector { + affinity_group: NameOrId::Id(_), + .. + } => { + Err(Error::invalid_request( + "when providing affinity_group as an ID project should not be specified", + )) + } + _ => { + Err(Error::invalid_request( + "affinity_group should either be UUID or project should be specified", + )) + } + } + } + + pub fn anti_affinity_group_lookup<'a>( + &'a self, + opctx: &'a OpContext, + anti_affinity_group_selector: params::AntiAffinityGroupSelector, + ) -> LookupResult> { + match anti_affinity_group_selector { + params::AntiAffinityGroupSelector { + anti_affinity_group: NameOrId::Id(id), + project: None + } => { + let anti_affinity_group = + LookupPath::new(opctx, &self.db_datastore).anti_affinity_group_id(id); + Ok(anti_affinity_group) + } + params::AntiAffinityGroupSelector { + anti_affinity_group: NameOrId::Name(name), + project: Some(project) + } => { + let anti_affinity_group = self + .project_lookup(opctx, params::ProjectSelector { project })? + .anti_affinity_group_name_owned(name.into()); + Ok(anti_affinity_group) + } + params::AntiAffinityGroupSelector { + anti_affinity_group: NameOrId::Id(_), + .. + } => { + Err(Error::invalid_request( + "when providing anti_affinity_group as an ID project should not be specified", + )) + } + _ => { + Err(Error::invalid_request( + "anti_affinity_group should either be UUID or project should be specified", + )) + } + } + } + + pub(crate) async fn affinity_group_list( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::ListChildren).await?; + + Ok(self + .db_datastore + .affinity_group_list(opctx, &authz_project, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn anti_affinity_group_list( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::ListChildren).await?; + + Ok(self + .db_datastore + .anti_affinity_group_list(opctx, &authz_project, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn affinity_group_create( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + affinity_group_params: params::AffinityGroupCreate, + ) -> CreateResult { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::CreateChild).await?; + + let affinity_group = + AffinityGroup::new(authz_project.id(), affinity_group_params); + self.db_datastore + .affinity_group_create(opctx, &authz_project, affinity_group) + .await + .map(Into::into) + } + + pub(crate) async fn anti_affinity_group_create( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + anti_affinity_group_params: params::AntiAffinityGroupCreate, + ) -> CreateResult { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::CreateChild).await?; + + let anti_affinity_group = AntiAffinityGroup::new( + authz_project.id(), + anti_affinity_group_params, + ); + self.db_datastore + .anti_affinity_group_create( + opctx, + &authz_project, + anti_affinity_group, + ) + .await + .map(Into::into) + } + + pub(crate) async fn affinity_group_update( + &self, + opctx: &OpContext, + group_lookup: &lookup::AffinityGroup<'_>, + updates: ¶ms::AffinityGroupUpdate, + ) -> UpdateResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Modify).await?; + self.db_datastore + .affinity_group_update(opctx, &authz_group, updates.clone().into()) + .await + .map(|g| g.into()) + } + + pub(crate) async fn anti_affinity_group_update( + &self, + opctx: &OpContext, + group_lookup: &lookup::AntiAffinityGroup<'_>, + updates: ¶ms::AntiAffinityGroupUpdate, + ) -> UpdateResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Modify).await?; + self.db_datastore + .anti_affinity_group_update( + opctx, + &authz_group, + updates.clone().into(), + ) + .await + .map(|g| g.into()) + } + + pub(crate) async fn affinity_group_delete( + &self, + opctx: &OpContext, + group_lookup: &lookup::AffinityGroup<'_>, + ) -> DeleteResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Delete).await?; + self.db_datastore.affinity_group_delete(opctx, &authz_group).await + } + + pub(crate) async fn anti_affinity_group_delete( + &self, + opctx: &OpContext, + group_lookup: &lookup::AntiAffinityGroup<'_>, + ) -> DeleteResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Delete).await?; + self.db_datastore.anti_affinity_group_delete(opctx, &authz_group).await + } + + pub(crate) async fn affinity_group_member_list( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_affinity_group) = affinity_group_lookup + .lookup_for(authz::Action::ListChildren) + .await?; + Ok(self + .db_datastore + .affinity_group_member_list(opctx, &authz_affinity_group, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn anti_affinity_group_member_list( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::ListChildren) + .await?; + Ok(self + .db_datastore + .anti_affinity_group_member_list( + opctx, + &authz_anti_affinity_group, + pagparams, + ) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn affinity_group_member_view( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .affinity_group_member_view(opctx, &authz_affinity_group, member) + .await + } + + pub(crate) async fn anti_affinity_group_member_view( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = + anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AntiAffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .anti_affinity_group_member_view( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } + + pub(crate) async fn affinity_group_member_add( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Modify).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .affinity_group_member_add( + opctx, + &authz_affinity_group, + member.clone(), + ) + .await?; + Ok(member) + } + + pub(crate) async fn anti_affinity_group_member_add( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AntiAffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .anti_affinity_group_member_add( + opctx, + &authz_anti_affinity_group, + member.clone(), + ) + .await?; + Ok(member) + } + + pub(crate) async fn affinity_group_member_delete( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result<(), Error> { + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Modify).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .affinity_group_member_delete(opctx, &authz_affinity_group, member) + .await + } + + pub(crate) async fn anti_affinity_group_member_delete( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result<(), Error> { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AntiAffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .anti_affinity_group_member_delete( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } +} diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index db87d7db7d7..08d84244d83 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -46,6 +46,7 @@ use uuid::Uuid; // The implementation of Nexus is large, and split into a number of submodules // by resource. mod address_lot; +mod affinity; mod allow_list; pub(crate) mod background; mod bfd; diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 887a6d35a65..b1cf5b59e5a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -15,7 +15,6 @@ use super::{ }; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; -use crate::app::Unimpl; use crate::context::ApiContext; use crate::external_api::shared; use dropshot::Body; @@ -55,6 +54,7 @@ use nexus_types::{ }, }; use omicron_common::api::external::http_pagination::data_page_params_for; +use omicron_common::api::external::http_pagination::id_pagination; use omicron_common::api::external::http_pagination::marker_for_id; use omicron_common::api::external::http_pagination::marker_for_name; use omicron_common::api::external::http_pagination::marker_for_name_or_id; @@ -2513,7 +2513,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_list( rqctx: RequestContext, - _query_params: Query>, + query_params: Query>, ) -> Result>, HttpError> { let apictx = rqctx.context(); @@ -2521,7 +2521,20 @@ impl NexusExternalApi for NexusExternalApiImpl { let nexus = &apictx.context.nexus; let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let groups = nexus + .affinity_group_list(&opctx, &project_lookup, &paginated_by) + .await?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + groups, + &marker_for_name_or_id, + )?)) }; apictx .context @@ -2532,16 +2545,28 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_view( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + + let (.., group) = nexus + .affinity_group_lookup(&opctx, group_selector)? + .fetch() + .await?; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + Ok(HttpResponseOk(group.into())) }; apictx .context @@ -2552,8 +2577,8 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_member_list( rqctx: RequestContext, - _query_params: Query>, - _path_params: Path, + query_params: Query>, + path_params: Path, ) -> Result>, HttpError> { let apictx = rqctx.context(); @@ -2561,7 +2586,30 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanById::from_query(&query)?; + let paginated_by = id_pagination(&pag_params, scan_params)?; + + let group_selector = params::AffinityGroupSelector { + project: scan_params.selector.project.clone(), + affinity_group: path.affinity_group, + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + let affinity_group_member_instances = nexus + .affinity_group_member_list( + &opctx, + &group_lookup, + &paginated_by, + ) + .await?; + Ok(HttpResponseOk(ScanById::results_page( + &query, + affinity_group_member_instances, + &marker_for_id, + )?)) }; apictx .context @@ -2572,16 +2620,42 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_member_instance_view( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let group = nexus + .affinity_group_member_view( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + + Ok(HttpResponseOk(group)) }; apictx .context @@ -2592,15 +2666,41 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_member_instance_add( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let member = nexus + .affinity_group_member_add( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseCreated(member)) }; apictx .context @@ -2611,15 +2711,40 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_member_instance_delete( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + nexus + .affinity_group_member_delete( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseDeleted()) }; apictx .context @@ -2630,15 +2755,25 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_create( rqctx: RequestContext, - _query_params: Query, - _new_affinity_group_params: TypedBody, + query_params: Query, + new_affinity_group_params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let query = query_params.into_inner(); + let project_lookup = nexus.project_lookup(&opctx, query)?; + let new_affinity_group = new_affinity_group_params.into_inner(); + let affinity_group = nexus + .affinity_group_create( + &opctx, + &project_lookup, + new_affinity_group, + ) + .await?; + Ok(HttpResponseCreated(affinity_group)) }; apictx .context @@ -2649,16 +2784,28 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_update( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, - _updated_group: TypedBody, + query_params: Query, + path_params: Path, + updated_group: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let updates = updated_group.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + let affinity_group = nexus + .affinity_group_update(&opctx, &group_lookup, &updates) + .await?; + Ok(HttpResponseOk(affinity_group)) }; apictx .context @@ -2669,15 +2816,24 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_delete( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + nexus.affinity_group_delete(&opctx, &group_lookup).await?; + Ok(HttpResponseDeleted()) }; apictx .context @@ -2688,7 +2844,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_list( rqctx: RequestContext, - _query_params: Query>, + query_params: Query>, ) -> Result>, HttpError> { let apictx = rqctx.context(); @@ -2696,7 +2852,24 @@ impl NexusExternalApi for NexusExternalApiImpl { let nexus = &apictx.context.nexus; let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let groups = nexus + .anti_affinity_group_list( + &opctx, + &project_lookup, + &paginated_by, + ) + .await?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + groups, + &marker_for_name_or_id, + )?)) }; apictx .context @@ -2707,15 +2880,28 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_view( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let query = query_params.into_inner(); + + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + + let (.., group) = nexus + .anti_affinity_group_lookup(&opctx, group_selector)? + .fetch() + .await?; + + Ok(HttpResponseOk(group.into())) }; apictx .context @@ -2726,8 +2912,8 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_member_list( rqctx: RequestContext, - _query_params: Query>, - _path_params: Path, + query_params: Query>, + path_params: Path, ) -> Result>, HttpError> { let apictx = rqctx.context(); @@ -2735,7 +2921,30 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanById::from_query(&query)?; + let paginated_by = id_pagination(&pag_params, scan_params)?; + + let group_selector = params::AntiAffinityGroupSelector { + project: scan_params.selector.project.clone(), + anti_affinity_group: path.anti_affinity_group, + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + let group_members = nexus + .anti_affinity_group_member_list( + &opctx, + &group_lookup, + &paginated_by, + ) + .await?; + Ok(HttpResponseOk(ScanById::results_page( + &query, + group_members, + &marker_for_id, + )?)) }; apictx .context @@ -2746,15 +2955,42 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_member_instance_view( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let group = nexus + .anti_affinity_group_member_view( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + + Ok(HttpResponseOk(group)) }; apictx .context @@ -2765,15 +3001,41 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_member_instance_add( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let member = nexus + .anti_affinity_group_member_add( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseCreated(member)) }; apictx .context @@ -2784,15 +3046,41 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_member_instance_delete( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + nexus + .anti_affinity_group_member_delete( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseDeleted()) }; apictx .context @@ -2803,8 +3091,8 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_create( rqctx: RequestContext, - _query_params: Query, - _new_anti_affinity_group_params: TypedBody< + query_params: Query, + new_anti_affinity_group_params: TypedBody< params::AntiAffinityGroupCreate, >, ) -> Result, HttpError> { @@ -2813,7 +3101,18 @@ impl NexusExternalApi for NexusExternalApiImpl { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let query = query_params.into_inner(); + let project_lookup = nexus.project_lookup(&opctx, query)?; + let new_anti_affinity_group = + new_anti_affinity_group_params.into_inner(); + let anti_affinity_group = nexus + .anti_affinity_group_create( + &opctx, + &project_lookup, + new_anti_affinity_group, + ) + .await?; + Ok(HttpResponseCreated(anti_affinity_group)) }; apictx .context @@ -2824,16 +3123,28 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_update( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, - _updated_group: TypedBody, + query_params: Query, + path_params: Path, + updated_group: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let updates = updated_group.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AntiAffinityGroupSelector { + project: query.project, + anti_affinity_group: path.anti_affinity_group, + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + let anti_affinity_group = nexus + .anti_affinity_group_update(&opctx, &group_lookup, &updates) + .await?; + Ok(HttpResponseOk(anti_affinity_group)) }; apictx .context @@ -2844,15 +3155,24 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_delete( rqctx: RequestContext, - _query_params: Query, - _path_params: Path, + query_params: Query, + path_params: Path, ) -> Result { let apictx = rqctx.context(); let handler = async { let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; - Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AntiAffinityGroupSelector { + project: query.project, + anti_affinity_group: path.anti_affinity_group, + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + nexus.anti_affinity_group_delete(&opctx, &group_lookup).await?; + Ok(HttpResponseDeleted()) }; apictx .context diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 9fe2b62a276..3ef1858cfa2 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -21,6 +21,8 @@ use nexus_types::external_api::shared::Baseboard; use nexus_types::external_api::shared::IdentityType; use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::views; +use nexus_types::external_api::views::AffinityGroup; +use nexus_types::external_api::views::AntiAffinityGroup; use nexus_types::external_api::views::Certificate; use nexus_types::external_api::views::FloatingIp; use nexus_types::external_api::views::InternetGateway; @@ -33,9 +35,11 @@ use nexus_types::external_api::views::VpcSubnet; use nexus_types::external_api::views::{Project, Silo, Vpc, VpcRouter}; use nexus_types::identity::Resource; use nexus_types::internal_api::params as internal_params; +use omicron_common::api::external::AffinityPolicy; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Disk; use omicron_common::api::external::Error; +use omicron_common::api::external::FailureDomain; use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Instance; @@ -581,6 +585,46 @@ where object_create_error(client, &url, body, status).await } +pub async fn create_affinity_group( + client: &ClientTestContext, + project_name: &str, + group_name: &str, +) -> AffinityGroup { + object_create( + &client, + format!("/v1/affinity-groups?project={}", &project_name).as_str(), + ¶ms::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("affinity group description"), + }, + policy: AffinityPolicy::Fail, + failure_domain: FailureDomain::Sled, + }, + ) + .await +} + +pub async fn create_anti_affinity_group( + client: &ClientTestContext, + project_name: &str, + group_name: &str, +) -> AntiAffinityGroup { + object_create( + &client, + format!("/v1/anti-affinity-groups?project={}", &project_name).as_str(), + ¶ms::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("affinity group description"), + }, + policy: AffinityPolicy::Fail, + failure_domain: FailureDomain::Sled, + }, + ) + .await +} + pub async fn create_vpc( client: &ClientTestContext, project_name: &str, diff --git a/nexus/tests/integration_tests/affinity.rs b/nexus/tests/integration_tests/affinity.rs new file mode 100644 index 00000000000..65ab96b0a90 --- /dev/null +++ b/nexus/tests/integration_tests/affinity.rs @@ -0,0 +1,895 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tests Affinity (and Anti-Affinity) Groups + +use dropshot::test_util::ClientTestContext; +use dropshot::HttpErrorResponseBody; +use http::StatusCode; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_default_ip_pool; +use nexus_test_utils::resource_helpers::create_instance_with; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::object_create; +use nexus_test_utils::resource_helpers::object_create_error; +use nexus_test_utils::resource_helpers::object_delete; +use nexus_test_utils::resource_helpers::object_delete_error; +use nexus_test_utils::resource_helpers::object_get; +use nexus_test_utils::resource_helpers::object_get_error; +use nexus_test_utils::resource_helpers::object_put; +use nexus_test_utils::resource_helpers::object_put_error; +use nexus_test_utils::resource_helpers::objects_list_page_authz; +use nexus_test_utils_macros::nexus_test; +use nexus_types::external_api::params; +use nexus_types::external_api::views::AffinityGroup; +use nexus_types::external_api::views::AntiAffinityGroup; +use nexus_types::external_api::views::Sled; +use nexus_types::external_api::views::SledInstance; +use omicron_common::api::external; +use omicron_common::api::external::AffinityGroupMember; +use omicron_common::api::external::AntiAffinityGroupMember; +use omicron_common::api::external::ObjectIdentity; +use std::collections::BTreeSet; +use std::marker::PhantomData; + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +// Simplifying mechanism for making calls to Nexus' external API +struct ApiHelper<'a> { + client: &'a ClientTestContext, +} + +// This is an extention of the "ApiHelper", with an opinion about: +// +// - What project (if any) is selected +// - Whether or not we're accessing affinity/anti-affinity groups +struct ProjectScopedApiHelper<'a, T> { + client: &'a ClientTestContext, + project: Option<&'a str>, + affinity_type: PhantomData, +} + +impl ProjectScopedApiHelper<'_, T> { + async fn create_stopped_instance( + &self, + instance_name: &str, + ) -> external::Instance { + create_instance_with( + &self.client, + &self.project.as_ref().expect("Need to specify project name"), + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::None, + // Disks= + Vec::::new(), + // External IPs= + Vec::::new(), + // Start= + false, + // Auto-restart policy= + None, + ) + .await + } + + async fn groups_list(&self) -> Vec { + let url = groups_url(T::URL_COMPONENT, self.project); + objects_list_page_authz(&self.client, &url).await.items + } + + async fn groups_list_expect_error( + &self, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = groups_url(T::URL_COMPONENT, self.project); + object_get_error(&self.client, &url, status).await + } + + async fn group_create(&self, group: &str) -> T::Group { + let url = groups_url(T::URL_COMPONENT, self.project); + let params = T::make_create_params(group); + object_create(&self.client, &url, ¶ms).await + } + + async fn group_create_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = groups_url(T::URL_COMPONENT, self.project); + let params = T::make_create_params(group); + object_create_error(&self.client, &url, ¶ms, status).await + } + + async fn group_get(&self, group: &str) -> T::Group { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_get(&self.client, &url).await + } + + async fn group_get_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_get_error(&self.client, &url, status).await + } + + async fn group_update(&self, group: &str) -> T::Group { + let url = group_url(T::URL_COMPONENT, self.project, group); + let params = T::make_update_params(); + object_put(&self.client, &url, ¶ms).await + } + + async fn group_update_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_url(T::URL_COMPONENT, self.project, group); + let params = T::make_update_params(); + object_put_error(&self.client, &url, ¶ms, status).await + } + + async fn group_delete(&self, group: &str) { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_delete(&self.client, &url).await + } + + async fn group_delete_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_delete_error(&self.client, &url, status).await + } + + async fn group_members_list(&self, group: &str) -> Vec { + let url = group_members_url(T::URL_COMPONENT, self.project, group); + objects_list_page_authz(&self.client, &url).await.items + } + + async fn group_members_list_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_members_url(T::URL_COMPONENT, self.project, group); + object_get_error(&self.client, &url, status).await + } + + async fn group_member_add(&self, group: &str, instance: &str) -> T::Member { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_create(&self.client, &url, &()).await + } + + async fn group_member_add_expect_error( + &self, + group: &str, + instance: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_create_error(&self.client, &url, &(), status).await + } + + async fn group_member_get(&self, group: &str, instance: &str) -> T::Member { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_get(&self.client, &url).await + } + + async fn group_member_get_expect_error( + &self, + group: &str, + instance: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_get_error(&self.client, &url, status).await + } + + async fn group_member_delete(&self, group: &str, instance: &str) { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_delete(&self.client, &url).await + } + + async fn group_member_delete_expect_error( + &self, + group: &str, + instance: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_delete_error(&self.client, &url, status).await + } +} + +impl<'a> ApiHelper<'a> { + fn new(client: &'a ClientTestContext) -> Self { + Self { client } + } + + fn use_project( + &'a self, + project: &'a str, + ) -> ProjectScopedApiHelper<'a, T> { + ProjectScopedApiHelper { + client: self.client, + project: Some(project), + affinity_type: PhantomData, + } + } + + fn no_project(&'a self) -> ProjectScopedApiHelper<'a, T> { + ProjectScopedApiHelper { + client: self.client, + project: None, + affinity_type: PhantomData, + } + } + + async fn create_project(&self, name: &str) { + create_project(&self.client, name).await; + } + + async fn sleds_list(&self) -> Vec { + let url = "/v1/system/hardware/sleds"; + objects_list_page_authz(&self.client, url).await.items + } + + async fn sled_instance_list(&self, sled: &str) -> Vec { + let url = format!("/v1/system/hardware/sleds/{sled}/instances"); + objects_list_page_authz(&self.client, &url).await.items + } + + async fn start_instance( + &self, + instance: &external::Instance, + ) -> external::Instance { + let uri = format!("/v1/instances/{}/start", instance.identity.id); + + NexusRequest::new( + RequestBuilder::new(&self.client, http::Method::POST, &uri) + .expect_status(Some(http::StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap_or_else(|e| { + panic!("failed to make \"POST\" request to {uri}: {e}") + }) + .parsed_body() + .unwrap() + } +} + +fn project_query_param_suffix(project: Option<&str>) -> String { + if let Some(project) = project { + format!("?project={project}") + } else { + String::new() + } +} + +/// Types and traits used by both affinity and anti-affinity groups. +/// +/// Use this trait if you're trying to test something which appilies to both +/// group types. Conversely, if you're trying to test behavior specific to one +/// type or the other, I recommend that you avoid making your tests generic. +trait AffinityGroupish { + /// The struct used to represent this group. + /// + /// Should be the result of GET-ing this group. + type Group: serde::de::DeserializeOwned + ObjectIdentity; + + /// The struct representing a single member within this group. + type Member: serde::de::DeserializeOwned; + + /// Parameters that can be used to construct this group as a part of a POST + /// request. + type CreateParams: serde::Serialize; + + /// Parameters that can be used to update this group as a part of a PUT + /// request. + type UpdateParams: serde::Serialize; + + const URL_COMPONENT: &'static str; + const RESOURCE_NAME: &'static str; + + fn make_create_params(group_name: &str) -> Self::CreateParams; + fn make_update_params() -> Self::UpdateParams; +} + +// Arbitrary text used to validate PUT calls to groups +const NEW_DESCRIPTION: &'static str = "Updated description"; + +struct AffinityType; + +impl AffinityGroupish for AffinityType { + type Group = AffinityGroup; + type Member = AffinityGroupMember; + type CreateParams = params::AffinityGroupCreate; + type UpdateParams = params::AffinityGroupUpdate; + + const URL_COMPONENT: &'static str = "affinity-groups"; + const RESOURCE_NAME: &'static str = "affinity-group"; + + fn make_create_params(group_name: &str) -> Self::CreateParams { + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("This is a description"), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + } + } + + fn make_update_params() -> Self::UpdateParams { + params::AffinityGroupUpdate { + identity: external::IdentityMetadataUpdateParams { + name: None, + description: Some(NEW_DESCRIPTION.to_string()), + }, + } + } +} + +struct AntiAffinityType; + +impl AffinityGroupish for AntiAffinityType { + type Group = AntiAffinityGroup; + type Member = AntiAffinityGroupMember; + type CreateParams = params::AntiAffinityGroupCreate; + type UpdateParams = params::AntiAffinityGroupUpdate; + + const URL_COMPONENT: &'static str = "anti-affinity-groups"; + const RESOURCE_NAME: &'static str = "anti-affinity-group"; + + fn make_create_params(group_name: &str) -> Self::CreateParams { + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("This is a description"), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + } + } + + fn make_update_params() -> Self::UpdateParams { + params::AntiAffinityGroupUpdate { + identity: external::IdentityMetadataUpdateParams { + name: None, + description: Some(NEW_DESCRIPTION.to_string()), + }, + } + } +} + +fn groups_url(ty: &str, project: Option<&str>) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}{query_params}") +} + +fn group_url(ty: &str, project: Option<&str>, group: &str) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}{query_params}") +} + +fn group_members_url(ty: &str, project: Option<&str>, group: &str) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}/members{query_params}") +} + +fn group_member_instance_url( + ty: &str, + project: Option<&str>, + group: &str, + instance: &str, +) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}/members/instance/{instance}{query_params}") +} + +#[nexus_test(extra_sled_agents = 2)] +async fn test_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + const EXPECTED_SLEDS: usize = 3; + const INSTANCE_COUNT: usize = EXPECTED_SLEDS; + + let api = ApiHelper::new(external_client); + + // Verify the expected sleds to begin with. + let sleds = api.sleds_list().await; + assert_eq!(sleds.len(), EXPECTED_SLEDS); + + // Verify that there are no instances on the sleds. + for sled in &sleds { + let sled_id = sled.identity.id.to_string(); + assert!(api.sled_instance_list(&sled_id).await.is_empty()); + } + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&external_client).await; + api.create_project(PROJECT_NAME).await; + + let project_api = api.use_project::(PROJECT_NAME); + + let mut instances = Vec::new(); + for i in 0..INSTANCE_COUNT { + instances.push( + project_api + .create_stopped_instance(&format!("test-instance-{i}")) + .await, + ); + } + + // When we start, we observe no affinity groups + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); + + // We can now create a group and observe it + let group = project_api.group_create(GROUP_NAME).await; + + // We can list it and also GET the group specifically + let groups = project_api.groups_list().await; + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].identity.id, group.identity.id); + + let observed_group = project_api.group_get(GROUP_NAME).await; + assert_eq!(observed_group.identity.id, group.identity.id); + + // List all members of the affinity group (expect nothing) + let members = project_api.group_members_list(GROUP_NAME).await; + assert!(members.is_empty()); + + // Add these instances to an affinity group + for instance in &instances { + project_api + .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) + .await; + } + + // List members again (expect all instances) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), instances.len()); + + // We can also list each member + for instance in &instances { + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .await; + } + + // Start the instances we created earlier. + // + // We don't actually care that they're "running" from the perspective of the + // simulated sled agent, we just want placement to be triggered from Nexus. + for instance in &instances { + api.start_instance(&instance).await; + } + + // Use a BTreeSet so we can ignore ordering when comparing instance + // placement. + let expected_instances = instances + .iter() + .map(|instance| instance.identity.id) + .collect::>(); + + // We expect that all sleds will be empty, except for one, which will have + // all the instances in our affinity group. + let mut empty_sleds = 0; + let mut populated_sleds = 0; + for sled in &sleds { + let observed_instances = api + .sled_instance_list(&sled.identity.id.to_string()) + .await + .into_iter() + .map(|sled_instance| sled_instance.identity.id) + .collect::>(); + + if !observed_instances.is_empty() { + assert_eq!(observed_instances, expected_instances); + populated_sleds += 1; + } else { + empty_sleds += 1; + } + } + assert_eq!(populated_sleds, 1); + assert_eq!(empty_sleds, 2); +} + +#[nexus_test(extra_sled_agents = 2)] +async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + const EXPECTED_SLEDS: usize = 3; + const INSTANCE_COUNT: usize = EXPECTED_SLEDS; + + let api = ApiHelper::new(external_client); + + // Verify the expected sleds to begin with. + let sleds = api.sleds_list().await; + assert_eq!(sleds.len(), EXPECTED_SLEDS); + + // Verify that there are no instances on the sleds. + for sled in &sleds { + let sled_id = sled.identity.id.to_string(); + assert!(api.sled_instance_list(&sled_id).await.is_empty()); + } + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&external_client).await; + api.create_project(PROJECT_NAME).await; + + let project_api = api.use_project::(PROJECT_NAME); + + let mut instances = Vec::new(); + for i in 0..INSTANCE_COUNT { + instances.push( + project_api + .create_stopped_instance(&format!("test-instance-{i}")) + .await, + ); + } + + // When we start, we observe no anti-affinity groups + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); + + // We can now create a group and observe it + let group = project_api.group_create(GROUP_NAME).await; + + // We can list it and also GET the group specifically + let groups = project_api.groups_list().await; + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].identity.id, group.identity.id); + + let observed_group = project_api.group_get(GROUP_NAME).await; + assert_eq!(observed_group.identity.id, group.identity.id); + + // List all members of the anti-affinity group (expect nothing) + let members = project_api.group_members_list(GROUP_NAME).await; + assert!(members.is_empty()); + + // Add these instances to the anti-affinity group + for instance in &instances { + project_api + .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) + .await; + } + + // List members again (expect all instances) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), instances.len()); + + // We can also list each member + for instance in &instances { + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .await; + } + + // Start the instances we created earlier. + // + // We don't actually care that they're "running" from the perspective of the + // simulated sled agent, we just want placement to be triggered from Nexus. + for instance in &instances { + api.start_instance(&instance).await; + } + + let mut expected_instances = instances + .iter() + .map(|instance| instance.identity.id) + .collect::>(); + + // We expect that each sled will have a since instance, as none of the + // instances will want to be anti-located from each other. + for sled in &sleds { + let observed_instances = api + .sled_instance_list(&sled.identity.id.to_string()) + .await + .into_iter() + .map(|sled_instance| sled_instance.identity.id) + .collect::>(); + + assert_eq!( + observed_instances.len(), + 1, + "All instances should be placed on distinct sleds" + ); + + assert!( + expected_instances.remove(&observed_instances[0]), + "The instance {} was observed on multiple sleds", + observed_instances[0] + ); + } + + assert!( + expected_instances.is_empty(), + "Did not find allocations for some instances: {expected_instances:?}" + ); +} + +#[nexus_test] +async fn test_affinity_group_crud(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + test_group_crud::(external_client).await; +} + +#[nexus_test] +async fn test_anti_affinity_group_crud(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + test_group_crud::(external_client).await; +} + +async fn test_group_crud(client: &ClientTestContext) { + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + + let api = ApiHelper::new(client); + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&client).await; + api.create_project(PROJECT_NAME).await; + + let project_api = api.use_project::(PROJECT_NAME); + + let instance = project_api.create_stopped_instance("test-instance").await; + + // When we start, we observe no affinity groups + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); + + // We can now create a group and observe it + project_api.group_create(GROUP_NAME).await; + let response = project_api + .group_create_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + assert_eq!( + response.message, + format!("already exists: {} \"{GROUP_NAME}\"", T::RESOURCE_NAME), + ); + + // We can modify the group itself + let group = project_api.group_update(GROUP_NAME).await; + assert_eq!(group.identity().description, NEW_DESCRIPTION); + + // List all members of the affinity group (expect nothing) + let members = project_api.group_members_list(GROUP_NAME).await; + assert!(members.is_empty()); + + // Add the instance to the affinity group + let instance_name = &instance.identity.name.to_string(); + project_api.group_member_add(GROUP_NAME, &instance_name).await; + let response = project_api + .group_member_add_expect_error( + GROUP_NAME, + &instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + response.message, + format!( + "already exists: {}-member \"{}\"", + T::RESOURCE_NAME, + instance.identity.id + ), + ); + + // List members again (expect the instance) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), 1); + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .await; + + // Delete the member, observe that it is gone + project_api.group_member_delete(GROUP_NAME, &instance_name).await; + project_api + .group_member_delete_expect_error( + GROUP_NAME, + &instance_name, + StatusCode::NOT_FOUND, + ) + .await; + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), 0); + project_api + .group_member_get_expect_error( + GROUP_NAME, + &instance_name, + StatusCode::NOT_FOUND, + ) + .await; + + // Delete the group, observe that it is gone + project_api.group_delete(GROUP_NAME).await; + project_api + .group_delete_expect_error(GROUP_NAME, StatusCode::NOT_FOUND) + .await; + project_api.group_get_expect_error(GROUP_NAME, StatusCode::NOT_FOUND).await; + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); +} + +#[nexus_test] +async fn test_affinity_group_project_selector( + cptestctx: &ControlPlaneTestContext, +) { + let external_client = &cptestctx.external_client; + test_group_project_selector::(external_client).await; +} + +#[nexus_test] +async fn test_anti_affinity_group_project_selector( + cptestctx: &ControlPlaneTestContext, +) { + let external_client = &cptestctx.external_client; + test_group_project_selector::(external_client).await; +} + +async fn test_group_project_selector( + client: &ClientTestContext, +) { + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + + let api = ApiHelper::new(client); + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&client).await; + api.create_project(PROJECT_NAME).await; + + // All requests use the "?project={PROJECT_NAME}" query parameter + let project_api = api.use_project::(PROJECT_NAME); + // All requests omit the project query parameter + let no_project_api = api.no_project::(); + + let instance = project_api.create_stopped_instance("test-instance").await; + + // We can only list groups within a project + no_project_api.groups_list_expect_error(StatusCode::BAD_REQUEST).await; + let _groups = project_api.groups_list().await; + + // We can only create a group within a project + no_project_api + .group_create_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + let group = project_api.group_create(GROUP_NAME).await; + + // Once we've created a group, we can access it by: + // + // - Project + Group Name, or + // - No Project + Group ID + // + // Other combinations are considered bad requests. + let group_id = group.identity().id.to_string(); + + project_api.group_get(GROUP_NAME).await; + no_project_api.group_get(&group_id).await; + project_api + .group_get_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_get_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + + // Same for listing members + project_api.group_members_list(GROUP_NAME).await; + no_project_api.group_members_list(&group_id).await; + project_api + .group_members_list_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_members_list_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + + // Same for updating the group + project_api.group_update(GROUP_NAME).await; + no_project_api.group_update(&group_id).await; + project_api + .group_update_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_update_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + + // Group Members can be added by name or UUID + let instance_name = instance.identity.name.as_str(); + let instance_id = instance.identity.id.to_string(); + project_api.group_member_add(GROUP_NAME, instance_name).await; + project_api.group_member_delete(GROUP_NAME, instance_name).await; + no_project_api.group_member_add(&group_id, &instance_id).await; + no_project_api.group_member_delete(&group_id, &instance_id).await; + + // Trying to use UUIDs with the project selector is invalid + project_api + .group_member_add_expect_error( + GROUP_NAME, + &instance_id, + StatusCode::BAD_REQUEST, + ) + .await; + project_api + .group_member_add_expect_error( + &group_id, + instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + + // Using any names without the project selector is invalid + no_project_api + .group_member_add_expect_error( + GROUP_NAME, + &instance_id, + StatusCode::BAD_REQUEST, + ) + .await; + no_project_api + .group_member_add_expect_error( + &group_id, + instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + no_project_api + .group_member_add_expect_error( + GROUP_NAME, + instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + + // Group deletion also prevents mixing {project, ID} and {no-project, name}. + project_api + .group_delete_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_delete_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + no_project_api.group_delete(&group_id).await; +} diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 656b4ba8266..cb145a840d0 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -24,8 +24,10 @@ use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::shared::Ipv4Range; use nexus_types::external_api::views::SledProvisionPolicy; use omicron_common::api::external::AddressLotKind; +use omicron_common::api::external::AffinityPolicy; use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::ByteCount; +use omicron_common::api::external::FailureDomain; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::InstanceCpuCount; @@ -151,6 +153,12 @@ pub static DEMO_PROJECT_URL_IMAGES: Lazy = Lazy::new(|| format!("/v1/images?project={}", *DEMO_PROJECT_NAME)); pub static DEMO_PROJECT_URL_INSTANCES: Lazy = Lazy::new(|| format!("/v1/instances?project={}", *DEMO_PROJECT_NAME)); +pub static DEMO_PROJECT_URL_AFFINITY_GROUPS: Lazy = + Lazy::new(|| format!("/v1/affinity-groups?project={}", *DEMO_PROJECT_NAME)); +pub static DEMO_PROJECT_URL_ANTI_AFFINITY_GROUPS: Lazy = + Lazy::new(|| { + format!("/v1/anti-affinity-groups?project={}", *DEMO_PROJECT_NAME) + }); pub static DEMO_PROJECT_URL_SNAPSHOTS: Lazy = Lazy::new(|| format!("/v1/snapshots?project={}", *DEMO_PROJECT_NAME)); pub static DEMO_PROJECT_URL_VPCS: Lazy = @@ -415,9 +423,99 @@ pub static DEMO_IMPORT_DISK_FINALIZE_URL: Lazy = Lazy::new(|| { ) }); +// Affinity/Anti- group used for testing + +pub static DEMO_AFFINITY_GROUP_NAME: Lazy = + Lazy::new(|| "demo-affinity-group".parse().unwrap()); +pub static DEMO_AFFINITY_GROUP_URL: Lazy = Lazy::new(|| { + format!( + "/v1/affinity-groups/{}?{}", + *DEMO_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) +}); +pub static DEMO_AFFINITY_GROUP_MEMBERS_URL: Lazy = Lazy::new(|| { + format!( + "/v1/affinity-groups/{}/members?{}", + *DEMO_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) +}); +pub static DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL: Lazy = + Lazy::new(|| { + format!( + "/v1/affinity-groups/{}/members/instance/{}?{}", + *DEMO_AFFINITY_GROUP_NAME, + *DEMO_STOPPED_INSTANCE_NAME, + *DEMO_PROJECT_SELECTOR + ) + }); +pub static DEMO_AFFINITY_GROUP_CREATE: Lazy = + Lazy::new(|| params::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_AFFINITY_GROUP_NAME.clone(), + description: String::from(""), + }, + policy: AffinityPolicy::Allow, + failure_domain: FailureDomain::Sled, + }); +pub static DEMO_AFFINITY_GROUP_UPDATE: Lazy = + Lazy::new(|| params::AffinityGroupUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: Some(String::from("an updated description")), + }, + }); + +pub static DEMO_ANTI_AFFINITY_GROUP_NAME: Lazy = + Lazy::new(|| "demo-anti-affinity-group".parse().unwrap()); +pub static DEMO_ANTI_AFFINITY_GROUPS_URL: Lazy = Lazy::new(|| { + format!("/v1/anti-affinity-groups?{}", *DEMO_PROJECT_SELECTOR) +}); +pub static DEMO_ANTI_AFFINITY_GROUP_URL: Lazy = Lazy::new(|| { + format!( + "/v1/anti-affinity-groups/{}?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) +}); +pub static DEMO_ANTI_AFFINITY_GROUP_MEMBERS_URL: Lazy = + Lazy::new(|| { + format!( + "/v1/anti-affinity-groups/{}/members?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) + }); +pub static DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL: Lazy = + Lazy::new(|| { + format!( + "/v1/anti-affinity-groups/{}/members/instance/{}?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, + *DEMO_STOPPED_INSTANCE_NAME, + *DEMO_PROJECT_SELECTOR + ) + }); +pub static DEMO_ANTI_AFFINITY_GROUP_CREATE: Lazy< + params::AntiAffinityGroupCreate, +> = Lazy::new(|| params::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_ANTI_AFFINITY_GROUP_NAME.clone(), + description: String::from(""), + }, + policy: AffinityPolicy::Allow, + failure_domain: FailureDomain::Sled, +}); +pub static DEMO_ANTI_AFFINITY_GROUP_UPDATE: Lazy< + params::AntiAffinityGroupUpdate, +> = Lazy::new(|| params::AntiAffinityGroupUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: Some(String::from("an updated description")), + }, +}); + // Instance used for testing pub static DEMO_INSTANCE_NAME: Lazy = Lazy::new(|| "demo-instance".parse().unwrap()); +pub static DEMO_STOPPED_INSTANCE_NAME: Lazy = + Lazy::new(|| "demo-stopped-instance".parse().unwrap()); pub static DEMO_INSTANCE_URL: Lazy = Lazy::new(|| { format!("/v1/instances/{}?{}", *DEMO_INSTANCE_NAME, *DEMO_PROJECT_SELECTOR) }); @@ -513,6 +611,26 @@ pub static DEMO_INSTANCE_CREATE: Lazy = start: true, auto_restart_policy: Default::default(), }); +pub static DEMO_STOPPED_INSTANCE_CREATE: Lazy = + Lazy::new(|| params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_STOPPED_INSTANCE_NAME.clone(), + description: String::from(""), + }, + ncpus: InstanceCpuCount(1), + memory: ByteCount::from_gibibytes_u32(16), + hostname: "demo-instance".parse().unwrap(), + user_data: vec![], + ssh_public_keys: Some(Vec::new()), + network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![params::ExternalIpCreate::Ephemeral { + pool: Some(DEMO_IP_POOL_NAME.clone().into()), + }], + disks: vec![], + boot_disk: None, + start: true, + auto_restart_policy: Default::default(), + }); pub static DEMO_INSTANCE_UPDATE: Lazy = Lazy::new(|| params::InstanceUpdate { boot_disk: None, @@ -1935,6 +2053,108 @@ pub static VERIFY_ENDPOINTS: Lazy> = Lazy::new(|| { ] }, + /* Affinity Groups */ + + VerifyEndpoint { + url: &DEMO_PROJECT_URL_AFFINITY_GROUPS, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_AFFINITY_GROUP_CREATE).unwrap() + ), + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_AFFINITY_GROUP_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Put( + serde_json::to_value(&*DEMO_AFFINITY_GROUP_UPDATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_AFFINITY_GROUP_MEMBERS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Post(serde_json::Value::Null), + ], + }, + + /* Anti-Affinity Groups */ + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUPS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_ANTI_AFFINITY_GROUP_CREATE).unwrap() + ), + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Put( + serde_json::to_value(&*DEMO_ANTI_AFFINITY_GROUP_UPDATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_MEMBERS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Post(serde_json::Value::Null) + ], + }, + /* Instances */ VerifyEndpoint { url: &DEMO_PROJECT_URL_INSTANCES, diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index dc404736cd1..6283b51f587 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -4,6 +4,7 @@ //! the way it is. mod address_lots; +mod affinity; mod allow_list; mod authn_http; mod authz; diff --git a/nexus/tests/integration_tests/projects.rs b/nexus/tests/integration_tests/projects.rs index d9752b1949f..dcd99de9042 100644 --- a/nexus/tests/integration_tests/projects.rs +++ b/nexus/tests/integration_tests/projects.rs @@ -9,11 +9,17 @@ use http::StatusCode; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_affinity_group; +use nexus_test_utils::resource_helpers::create_anti_affinity_group; +use nexus_test_utils::resource_helpers::create_default_ip_pool; +use nexus_test_utils::resource_helpers::create_disk; use nexus_test_utils::resource_helpers::create_floating_ip; -use nexus_test_utils::resource_helpers::{ - create_default_ip_pool, create_disk, create_project, create_vpc, - object_create, project_get, projects_list, DiskTest, -}; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::create_vpc; +use nexus_test_utils::resource_helpers::object_create; +use nexus_test_utils::resource_helpers::project_get; +use nexus_test_utils::resource_helpers::projects_list; +use nexus_test_utils::resource_helpers::DiskTest; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; use nexus_types::external_api::views; @@ -392,3 +398,67 @@ async fn test_project_deletion_with_vpc(cptestctx: &ControlPlaneTestContext) { .unwrap(); delete_project(&project_url, &client).await; } + +#[nexus_test] +async fn test_project_deletion_with_affinity_group( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + // Create a project that we'll use for testing. + let name = "springfield-squidport"; + let project_url = format!("/v1/projects/{}", name); + + create_project(&client, &name).await; + delete_project_default_subnet(&name, &client).await; + delete_project_default_vpc(&name, &client).await; + + let group_name = "just-rainsticks"; + create_affinity_group(&client, name, group_name).await; + + assert_eq!( + "project to be deleted contains an affinity group: just-rainsticks", + delete_project_expect_fail(&project_url, &client).await, + ); + + let group_url = + format!("/v1/affinity-groups/{group_name}?project={}", name); + NexusRequest::object_delete(client, &group_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + delete_project(&project_url, &client).await; +} + +#[nexus_test] +async fn test_project_deletion_with_anti_affinity_group( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + // Create a project that we'll use for testing. + let name = "springfield-squidport"; + let project_url = format!("/v1/projects/{}", name); + + create_project(&client, &name).await; + delete_project_default_subnet(&name, &client).await; + delete_project_default_vpc(&name, &client).await; + + let group_name = "just-rainsticks"; + create_anti_affinity_group(&client, name, group_name).await; + + assert_eq!( + "project to be deleted contains an anti affinity group: just-rainsticks", + delete_project_expect_fail(&project_url, &client).await, + ); + + let group_url = + format!("/v1/anti-affinity-groups/{group_name}?project={}", name); + NexusRequest::object_delete(client, &group_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + delete_project(&project_url, &client).await; +} diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index 45f87c96ce0..26c0711f90f 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -292,6 +292,37 @@ static SETUP_REQUESTS: Lazy> = Lazy::new(|| { body: serde_json::to_value(&*DEMO_INSTANCE_CREATE).unwrap(), id_routes: vec!["/v1/instances/{id}"], }, + // Create a stopped Instance in the Project + SetupReq::Post { + url: &DEMO_PROJECT_URL_INSTANCES, + body: serde_json::to_value(&*DEMO_STOPPED_INSTANCE_CREATE).unwrap(), + id_routes: vec!["/v1/instances/{id}"], + }, + // Create an affinity group in the Project + SetupReq::Post { + url: &DEMO_PROJECT_URL_AFFINITY_GROUPS, + body: serde_json::to_value(&*DEMO_AFFINITY_GROUP_CREATE).unwrap(), + id_routes: vec!["/v1/affinity-groups/{id}"], + }, + // Add a member to the affinity group + SetupReq::Post { + url: &DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + body: serde_json::Value::Null, + id_routes: vec![], + }, + // Create an anti-affinity group in the Project + SetupReq::Post { + url: &DEMO_PROJECT_URL_ANTI_AFFINITY_GROUPS, + body: serde_json::to_value(&*DEMO_ANTI_AFFINITY_GROUP_CREATE) + .unwrap(), + id_routes: vec!["/v1/anti-affinity-groups/{id}"], + }, + // Add a member to the anti-affinity group + SetupReq::Post { + url: &DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + body: serde_json::Value::Null, + id_routes: vec![], + }, // Lookup the previously created NIC SetupReq::Get { url: &DEMO_INSTANCE_NIC_URL, diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index c943728beb4..8a639f1224c 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -1,22 +1,10 @@ API endpoints with no coverage in authz tests: probe_delete (delete "/experimental/v1/probes/{probe}") -affinity_group_delete (delete "/v1/affinity-groups/{affinity_group}") -affinity_group_member_instance_delete (delete "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") -anti_affinity_group_delete (delete "/v1/anti-affinity-groups/{anti_affinity_group}") -anti_affinity_group_member_instance_delete (delete "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") probe_list (get "/experimental/v1/probes") probe_view (get "/experimental/v1/probes/{probe}") support_bundle_download (get "/experimental/v1/system/support-bundles/{support_bundle}/download") support_bundle_download_file (get "/experimental/v1/system/support-bundles/{support_bundle}/download/{file}") support_bundle_index (get "/experimental/v1/system/support-bundles/{support_bundle}/index") -affinity_group_list (get "/v1/affinity-groups") -affinity_group_view (get "/v1/affinity-groups/{affinity_group}") -affinity_group_member_list (get "/v1/affinity-groups/{affinity_group}/members") -affinity_group_member_instance_view (get "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") -anti_affinity_group_list (get "/v1/anti-affinity-groups") -anti_affinity_group_view (get "/v1/anti-affinity-groups/{anti_affinity_group}") -anti_affinity_group_member_list (get "/v1/anti-affinity-groups/{anti_affinity_group}/members") -anti_affinity_group_member_instance_view (get "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") ping (get "/v1/ping") networking_switch_port_lldp_neighbors (get "/v1/system/hardware/rack-switch-port/{rack_id}/{switch_location}/{port}/lldp/neighbors") networking_switch_port_lldp_config_view (get "/v1/system/hardware/switch-port/{port}/lldp/config") @@ -28,12 +16,6 @@ device_auth_confirm (post "/device/confirm") device_access_token (post "/device/token") probe_create (post "/experimental/v1/probes") login_saml (post "/login/{silo_name}/saml/{provider_name}") -affinity_group_create (post "/v1/affinity-groups") -affinity_group_member_instance_add (post "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") -anti_affinity_group_create (post "/v1/anti-affinity-groups") -anti_affinity_group_member_instance_add (post "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") login_local (post "/v1/login/{silo_name}/local") logout (post "/v1/logout") networking_switch_port_lldp_config_update (post "/v1/system/hardware/switch-port/{port}/lldp/config") -affinity_group_update (put "/v1/affinity-groups/{affinity_group}") -anti_affinity_group_update (put "/v1/anti-affinity-groups/{anti_affinity_group}") From 161f9d60759521080dcb84736e9077c631add27a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 13:04:02 -0800 Subject: [PATCH 06/84] fix policy tests --- .../src/policy_test/resource_builder.rs | 2 + nexus/db-queries/src/policy_test/resources.rs | 17 ++++ nexus/db-queries/tests/output/authz-roles.out | 84 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/nexus/db-queries/src/policy_test/resource_builder.rs b/nexus/db-queries/src/policy_test/resource_builder.rs index b6d7d97553e..310c11adf3c 100644 --- a/nexus/db-queries/src/policy_test/resource_builder.rs +++ b/nexus/db-queries/src/policy_test/resource_builder.rs @@ -243,6 +243,8 @@ macro_rules! impl_dyn_authorized_resource_for_resource { } impl_dyn_authorized_resource_for_resource!(authz::AddressLot); +impl_dyn_authorized_resource_for_resource!(authz::AffinityGroup); +impl_dyn_authorized_resource_for_resource!(authz::AntiAffinityGroup); impl_dyn_authorized_resource_for_resource!(authz::Blueprint); impl_dyn_authorized_resource_for_resource!(authz::Certificate); impl_dyn_authorized_resource_for_resource!(authz::DeviceAccessToken); diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index 6ee92e167cf..04655410533 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -300,6 +300,21 @@ async fn make_project( LookupType::ByName(vpc1_name.clone()), ); + let affinity_group_name = format!("{}-affinity-group1", project_name); + let affinity_group = authz::AffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(affinity_group_name.clone()), + ); + + let anti_affinity_group_name = + format!("{}-anti-affinity-group1", project_name); + let anti_affinity_group = authz::AntiAffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(anti_affinity_group_name.clone()), + ); + let instance_name = format!("{}-instance1", project_name); let instance = authz::Instance::new( project.clone(), @@ -313,6 +328,8 @@ async fn make_project( Uuid::new_v4(), LookupType::ByName(disk_name.clone()), )); + builder.new_resource(affinity_group.clone()); + builder.new_resource(anti_affinity_group.clone()); builder.new_resource(instance.clone()); builder.new_resource(authz::InstanceNetworkInterface::new( instance, diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 4b24e649ccb..e0d43250d13 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -306,6 +306,34 @@ resource: Disk "silo1-proj1-disk1" silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj1-instance1" USER Q R LC RP M MP CC D @@ -474,6 +502,34 @@ resource: Disk "silo1-proj2-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj2-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj2-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj2-instance1" USER Q R LC RP M MP CC D @@ -810,6 +866,34 @@ resource: Disk "silo2-proj1-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo2-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo2-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo2-proj1-instance1" USER Q R LC RP M MP CC D From 8dc0825fb4834c9b25030b4083efc59e13820c24 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 13:19:07 -0800 Subject: [PATCH 07/84] fmt --- nexus/src/external_api/http_entrypoints.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 906684e7825..887a6d35a65 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -13,9 +13,9 @@ use super::{ Utilization, Vpc, VpcRouter, VpcSubnet, }, }; -use crate::app::Unimpl; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; +use crate::app::Unimpl; use crate::context::ApiContext; use crate::external_api::shared; use dropshot::Body; From 4e9cebcf72fb8d85c2319ca61930721b9eca5bdd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 14:58:56 -0800 Subject: [PATCH 08/84] tags --- nexus/external-api/src/lib.rs | 2 +- openapi/nexus.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 7daa5514619..ad637c9ca41 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -75,7 +75,7 @@ const PUT_UPDATE_REPOSITORY_MAX_BYTES: usize = 4 * GIB; policy = EndpointTagPolicy::ExactlyOne, tags = { "affinity" = { - description = "Affinity groups give control over instance placement.", + description = "Affinity and anti-affinity groups give control over instance placement.", external_docs = { url = "http://docs.oxide.computer/api/affinity" } diff --git a/openapi/nexus.json b/openapi/nexus.json index 2e4ef3ac597..aac7aab1051 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -24558,7 +24558,7 @@ "tags": [ { "name": "affinity", - "description": "Affinity groups give control over instance placement.", + "description": "Affinity and anti-affinity groups give control over instance placement.", "externalDocs": { "url": "http://docs.oxide.computer/api/affinity" } From 6cfca2dc198d80cc08daaaab0785729de2f73ee9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 16:06:14 -0800 Subject: [PATCH 09/84] doc comments --- common/src/api/external/mod.rs | 13 +++++++++++++ openapi/nexus.json | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 98832487d5a..210953d49eb 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1318,6 +1318,9 @@ pub enum InstanceAutoRestartPolicy { // AFFINITY GROUPS +/// Affinity policy used to describe "what to do when a request cannot be satisfied" +/// +/// Used for both Affinity and Anti-Affinity Groups #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AffinityPolicy { @@ -1338,9 +1341,14 @@ pub enum FailureDomain { Sled, } +/// A member of an Affinity Group +/// +/// Membership in a group is not exclusive - members may belong to multiple +/// affinity / anti-affinity groups. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AffinityGroupMember { + /// An instance belonging to this group, identified by UUID. Instance(Uuid), } @@ -1352,9 +1360,14 @@ impl SimpleIdentity for AffinityGroupMember { } } +/// A member of an Anti-Affinity Group +/// +/// Membership in a group is not exclusive - members may belong to multiple +/// affinity / anti-affinity groups. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { + /// An instance belonging to this group, identified by UUID. Instance(Uuid), } diff --git a/openapi/nexus.json b/openapi/nexus.json index aac7aab1051..bc8abb272c2 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12205,8 +12205,10 @@ ] }, "AffinityGroupMember": { + "description": "A member of an Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { + "description": "An instance belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { @@ -12288,6 +12290,7 @@ } }, "AffinityPolicy": { + "description": "Affinity policy used to describe \"what to do when a request cannot be satisfied\"\n\nUsed for both Affinity and Anti-Affinity Groups", "oneOf": [ { "description": "If the affinity request cannot be satisfied, allow it anyway.\n\nThis enables a \"best-effort\" attempt to satisfy the affinity policy.", @@ -12482,8 +12485,10 @@ ] }, "AntiAffinityGroupMember": { + "description": "A member of an Anti-Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { + "description": "An instance belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { From 4b08032dc3c887cf4512518705d80f22b50b0a48 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:25:53 -0800 Subject: [PATCH 10/84] typed UUID --- common/src/api/external/mod.rs | 10 ++++++---- openapi/nexus.json | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 210953d49eb..f3234d74b99 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -23,6 +23,8 @@ use dropshot::HttpError; pub use dropshot::PaginationOrder; pub use error::*; use futures::stream::BoxStream; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; use oxnet::IpNet; use oxnet::Ipv4Net; use parse_display::Display; @@ -1349,13 +1351,13 @@ pub enum FailureDomain { #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AffinityGroupMember { /// An instance belonging to this group, identified by UUID. - Instance(Uuid), + Instance(InstanceUuid), } impl SimpleIdentity for AffinityGroupMember { fn id(&self) -> Uuid { match self { - AffinityGroupMember::Instance(id) => *id, + AffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), } } } @@ -1368,13 +1370,13 @@ impl SimpleIdentity for AffinityGroupMember { #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { /// An instance belonging to this group, identified by UUID. - Instance(Uuid), + Instance(InstanceUuid), } impl SimpleIdentity for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { - AntiAffinityGroupMember::Instance(id) => *id, + AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), } } } diff --git a/openapi/nexus.json b/openapi/nexus.json index bc8abb272c2..49b3c682465 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12218,8 +12218,7 @@ ] }, "value": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/TypedUuidForInstanceKind" } }, "required": [ @@ -12498,8 +12497,7 @@ ] }, "value": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/TypedUuidForInstanceKind" } }, "required": [ @@ -23055,6 +23053,10 @@ } } }, + "TypedUuidForInstanceKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForSupportBundleKind": { "type": "string", "format": "uuid" From 900f09c6a3acef149867adedff51fa0ab3324fb4 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:29:07 -0800 Subject: [PATCH 11/84] Typed UUID --- nexus/db-model/src/affinity.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index df97e7d1633..c2c4d30e224 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -23,7 +23,6 @@ use omicron_uuid_kinds::AffinityGroupKind; use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::AntiAffinityGroupKind; use omicron_uuid_kinds::AntiAffinityGroupUuid; -use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceKind; use omicron_uuid_kinds::InstanceUuid; use uuid::Uuid; @@ -231,7 +230,7 @@ impl AffinityGroupInstanceMembership { impl From for external::AffinityGroupMember { fn from(member: AffinityGroupInstanceMembership) -> Self { - Self::Instance(member.instance_id.into_untyped_uuid()) + Self::Instance(member.instance_id.into()) } } @@ -255,6 +254,6 @@ impl From for external::AntiAffinityGroupMember { fn from(member: AntiAffinityGroupInstanceMembership) -> Self { - Self::Instance(member.instance_id.into_untyped_uuid()) + Self::Instance(member.instance_id.into()) } } From f2ebe31df00ef40a655d070288c467c2c23398a9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:45:01 -0800 Subject: [PATCH 12/84] Typed UUID --- nexus/db-queries/src/db/datastore/affinity.rs | 154 ++++++++++++------ 1 file changed, 106 insertions(+), 48 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index e09a0e43406..78690c746b5 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -409,7 +409,7 @@ impl DataStore { use db::schema::affinity_group_instance_membership::dsl; dsl::affinity_group_instance_membership .filter(dsl::group_id.eq(authz_affinity_group.id())) - .filter(dsl::instance_id.eq(instance_id)) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) .select(AffinityGroupInstanceMembership::as_select()) .get_result_async(&*conn) .await @@ -419,7 +419,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::AffinityGroupMember, - LookupType::by_id(instance_id), + LookupType::by_id(instance_id.into_untyped_uuid()), ), ) }) @@ -441,7 +441,7 @@ impl DataStore { use db::schema::anti_affinity_group_instance_membership::dsl; dsl::anti_affinity_group_instance_membership .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter(dsl::instance_id.eq(instance_id)) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) .select(AntiAffinityGroupInstanceMembership::as_select()) .get_result_async(&*conn) .await @@ -451,7 +451,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::AntiAffinityGroupMember, - LookupType::by_id(instance_id), + LookupType::by_id(instance_id.into_untyped_uuid()), ), ) }) @@ -508,7 +508,7 @@ impl DataStore { // issues, so we do the lookup manually. let instance_state = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::state) .first_async(&conn) .await @@ -518,7 +518,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -543,7 +543,7 @@ impl DataStore { diesel::insert_into(membership_dsl::affinity_group_instance_membership) .values(AffinityGroupInstanceMembership::new( AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), - InstanceUuid::from_untyped_uuid(instance_id), + instance_id, )) .execute_async(&conn) .await @@ -616,7 +616,7 @@ impl DataStore { // Check that the instance exists, and has no VMM let instance_state = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::state) .first_async(&conn) .await @@ -626,7 +626,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -651,7 +651,7 @@ impl DataStore { diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) .values(AntiAffinityGroupInstanceMembership::new( AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), - InstanceUuid::from_untyped_uuid(instance_id), + instance_id, )) .execute_async(&conn) .await @@ -752,7 +752,7 @@ impl DataStore { // Check that the instance exists instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::id) .first_async::(&conn) .await @@ -762,7 +762,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -770,7 +770,7 @@ impl DataStore { let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) - .filter(membership_dsl::instance_id.eq(instance_id)) + .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) .execute_async(&conn) .await .map_err(|e| { @@ -779,7 +779,7 @@ impl DataStore { }) })?; if rows == 0 { - return Err(err.bail(LookupType::ById(instance_id).into_not_found( + return Err(err.bail(LookupType::ById(instance_id.into_untyped_uuid()).into_not_found( ResourceType::AffinityGroupMember, ))); } @@ -841,7 +841,7 @@ impl DataStore { // Check that the instance exists instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::id) .first_async::(&conn) .await @@ -851,7 +851,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -859,7 +859,7 @@ impl DataStore { let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter(membership_dsl::instance_id.eq(instance_id)) + .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) .execute_async(&conn) .await .map_err(|e| { @@ -868,7 +868,7 @@ impl DataStore { }) })?; if rows == 0 { - return Err(err.bail(LookupType::ById(instance_id).into_not_found( + return Err(err.bail(LookupType::ById(instance_id.into_untyped_uuid()).into_not_found( ResourceType::AntiAffinityGroupMember, ))); } @@ -1487,7 +1487,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1499,7 +1501,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1508,7 +1512,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1577,7 +1583,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1589,7 +1597,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1598,7 +1608,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1668,7 +1680,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err( @@ -1688,7 +1702,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1703,7 +1719,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1713,7 +1731,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1783,7 +1803,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance(InstanceUuid::from_untyped_uuid(instance.id())), ) .await .expect_err( @@ -1803,7 +1823,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1818,7 +1840,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1828,7 +1852,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1894,7 +1920,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1966,7 +1994,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2041,7 +2071,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2121,7 +2153,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2227,7 +2261,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2259,7 +2295,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2383,7 +2421,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2415,7 +2455,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2500,7 +2542,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2510,7 +2554,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); @@ -2546,7 +2592,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2554,7 +2602,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); @@ -2619,7 +2669,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2629,7 +2681,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); @@ -2665,7 +2719,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2673,7 +2729,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); From 1326116348edc10f67429205c2312ffd21e41cc8 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:48:30 -0800 Subject: [PATCH 13/84] UUID typing --- nexus/src/app/affinity.rs | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 894f29416e7..ca8d0c708cd 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -21,6 +21,8 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; impl super::Nexus { pub fn affinity_group_lookup<'a>( @@ -279,8 +281,9 @@ impl super::Nexus { affinity_group_lookup.lookup_for(authz::Action::Read).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = - external::AffinityGroupMember::Instance(authz_instance.id()); + let member = external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ); self.db_datastore .affinity_group_member_view(opctx, &authz_affinity_group, member) @@ -297,8 +300,9 @@ impl super::Nexus { anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = - external::AntiAffinityGroupMember::Instance(authz_instance.id()); + let member = external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ); self.db_datastore .anti_affinity_group_member_view( @@ -319,8 +323,9 @@ impl super::Nexus { affinity_group_lookup.lookup_for(authz::Action::Modify).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = - external::AffinityGroupMember::Instance(authz_instance.id()); + let member = external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ); self.db_datastore .affinity_group_member_add( @@ -343,8 +348,9 @@ impl super::Nexus { .await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = - external::AntiAffinityGroupMember::Instance(authz_instance.id()); + let member = external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ); self.db_datastore .anti_affinity_group_member_add( @@ -366,8 +372,9 @@ impl super::Nexus { affinity_group_lookup.lookup_for(authz::Action::Modify).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = - external::AffinityGroupMember::Instance(authz_instance.id()); + let member = external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ); self.db_datastore .affinity_group_member_delete(opctx, &authz_affinity_group, member) @@ -385,8 +392,9 @@ impl super::Nexus { .await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = - external::AntiAffinityGroupMember::Instance(authz_instance.id()); + let member = external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(authz_instance.id()), + ); self.db_datastore .anti_affinity_group_member_delete( From aba9596a0d1ce624ce59f8583ca7c687bc35f7f9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:59:07 -0800 Subject: [PATCH 14/84] comments --- nexus/db-model/src/affinity.rs | 2 +- schema/crdb/dbinit.sql | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index c2c4d30e224..5309ac95275 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/5.0/. -// Copyright 2024 Oxide Computer Company +// Copyright 2025 Oxide Computer Company //! Database representation of affinity and anti-affinity groups diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a2fa3d48e23..b8e110788d6 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -245,10 +245,15 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( kind omicron.public.sled_resource_kind NOT NULL, -- The UUID of an instance, if this resource belongs to an instance. + -- + -- This should eventually become NOT NULL for all instances, but is + -- still nullable for backwards compatibility purposes. Specifically, + -- the "instance start" saga can create rows in this table before creating + -- rows for "omicron.public.vmm", which we would use for back-filling. + -- If we tried to backfill + make this column non-nullable while that saga + -- was mid-execution, we would still have some rows in this table with nullable + -- values that would be more complex to fix. instance_id UUID - - -- TODO Add constraint that if kind is instance, instance_id is not NULL? - -- Or will that break backwards compatibility? ); -- Allow looking up all resources which reside on a sled From 1ad01010bcd8e59581a267463b3efb671a748343 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 16:00:03 -0800 Subject: [PATCH 15/84] review feedback --- common/src/api/external/mod.rs | 6 + nexus/db-queries/src/db/datastore/affinity.rs | 231 ++++++++++-------- 2 files changed, 139 insertions(+), 98 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f3234d74b99..f4b657a1a59 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -294,6 +294,12 @@ impl<'a> From<&'a Name> for &'a str { } } +impl From for String { + fn from(name: Name) -> Self { + name.0 + } +} + /// `Name` instances are comparable like Strings, primarily so that they can /// be used as keys in trees. impl PartialEq for Name diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 78690c746b5..5cd7c579105 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -20,7 +20,6 @@ use crate::db::model::AffinityGroupUpdate; use crate::db::model::AntiAffinityGroup; use crate::db::model::AntiAffinityGroupInstanceMembership; use crate::db::model::AntiAffinityGroupUpdate; -use crate::db::model::InstanceState; use crate::db::model::Name; use crate::db::model::Project; use crate::db::pagination::paginated; @@ -477,6 +476,7 @@ impl DataStore { use db::schema::affinity_group::dsl as group_dsl; use db::schema::affinity_group_instance_membership::dsl as membership_dsl; use db::schema::instance::dsl as instance_dsl; + use db::schema::sled_resource::dsl as resource_dsl; async move { // Check that the group exists @@ -497,7 +497,8 @@ impl DataStore { }) })?; - // Check that the instance exists, and has no VMM + // Check that the instance exists, and has no sled + // reservation. // // NOTE: I'd prefer to use the "LookupPath" infrastructure // to look up the path, but that API does not give the @@ -506,11 +507,12 @@ impl DataStore { // Looking up the instance on a different database // connection than the transaction risks several concurrency // issues, so we do the lookup manually. - let instance_state = instance_dsl::instance + + let _check_instance_exists = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::state) - .first_async(&conn) + .select(instance_dsl::id) + .get_result_async::(&conn) .await .map_err(|e| { err.bail_retryable_or_else(e, |e| { @@ -523,21 +525,33 @@ impl DataStore { ) }) })?; + let has_reservation: bool = diesel::select( + diesel::dsl::exists( + resource_dsl::sled_resource + .filter(resource_dsl::instance_id.eq(instance_id.into_untyped_uuid())) + ) + ).get_result_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + }) + })?; // NOTE: It may be possible to add non-stopped instances to // affinity groups, depending on where they have already // been placed. However, only operating on "stopped" // instances is much easier to work with, as it does not // require any understanding of the group policy. - match instance_state { - InstanceState::NoVmm => (), - other => { - return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to affinity group in state: {other}" - ) - ))); - }, + if has_reservation { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to affinity group with reservation" + ) + ))); } diesel::insert_into(membership_dsl::affinity_group_instance_membership) @@ -593,6 +607,7 @@ impl DataStore { use db::schema::anti_affinity_group::dsl as group_dsl; use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; use db::schema::instance::dsl as instance_dsl; + use db::schema::sled_resource::dsl as resource_dsl; async move { // Check that the group exists @@ -613,12 +628,13 @@ impl DataStore { }) })?; - // Check that the instance exists, and has no VMM - let instance_state = instance_dsl::instance + // Check that the instance exists, and has no sled + // reservation. + let _check_instance_exists = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::state) - .first_async(&conn) + .select(instance_dsl::id) + .get_result_async::(&conn) .await .map_err(|e| { err.bail_retryable_or_else(e, |e| { @@ -631,21 +647,33 @@ impl DataStore { ) }) })?; + let has_reservation: bool = diesel::select( + diesel::dsl::exists( + resource_dsl::sled_resource + .filter(resource_dsl::instance_id.eq(instance_id.into_untyped_uuid())) + ) + ).get_result_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + }) + })?; // NOTE: It may be possible to add non-stopped instances to - // anti affinity groups, depending on where they have already + // affinity groups, depending on where they have already // been placed. However, only operating on "stopped" // instances is much easier to work with, as it does not // require any understanding of the group policy. - match instance_state { - InstanceState::NoVmm => (), - other => { - return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to anti-affinity group in state: {other}" - ) - ))); - }, + if has_reservation { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to anti-affinity group with reservation" + ) + ))); } diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) @@ -728,7 +756,6 @@ impl DataStore { let err = err.clone(); use db::schema::affinity_group::dsl as group_dsl; use db::schema::affinity_group_instance_membership::dsl as membership_dsl; - use db::schema::instance::dsl as instance_dsl; async move { // Check that the group exists @@ -749,25 +776,6 @@ impl DataStore { }) })?; - // Check that the instance exists - instance_dsl::instance - .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::Instance, - LookupType::ById(instance_id.into_untyped_uuid()) - ), - ) - }) - })?; - let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) @@ -817,7 +825,6 @@ impl DataStore { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; - use db::schema::instance::dsl as instance_dsl; async move { // Check that the group exists @@ -838,25 +845,6 @@ impl DataStore { }) })?; - // Check that the instance exists - instance_dsl::instance - .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::Instance, - LookupType::ById(instance_id.into_untyped_uuid()) - ), - ) - }) - })?; - let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) @@ -893,12 +881,16 @@ mod tests { use crate::db::lookup::LookupPath; use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::Instance; + use nexus_db_model::Resources; + use nexus_db_model::SledResource; use nexus_types::external_api::params; use omicron_common::api::external::{ self, ByteCount, DataPageParams, IdentityMetadataCreateParams, }; use omicron_test_utils::dev; use omicron_uuid_kinds::InstanceUuid; + use omicron_uuid_kinds::PropolisUuid; + use omicron_uuid_kinds::SledUuid; use std::num::NonZeroU32; // Helper function for creating a project @@ -1006,14 +998,6 @@ mod tests { instance } - // Helper for explicitly modifying instance state. - // - // The interaction we typically use to create and modify instance state - // is more complex in production, since it's the result of a back-and-forth - // between Nexus and Sled Agent, using carefully crafted rcgen values. - // - // Here, we just set the value of state explicitly. Be warned, there - // are no guardrails! async fn set_instance_state_stopped( datastore: &DataStore, instance: uuid::Uuid, @@ -1032,16 +1016,28 @@ mod tests { .unwrap(); } - async fn set_instance_state_running( + // Helper for explicitly modifying sled resource usage + // + // The interaction we typically use to create and modify instance state + // is more complex in production, using a real allocation algorithm. + // + // Here, we just set the value of state explicitly. Be warned, there + // are no guardrails! + async fn allocate_instance_reservation( datastore: &DataStore, - instance: uuid::Uuid, + instance: InstanceUuid, ) { - use db::schema::instance::dsl; - diesel::update(dsl::instance) - .filter(dsl::id.eq(instance)) - .set(( - dsl::state.eq(db::model::InstanceState::Vmm), - dsl::active_propolis_id.eq(uuid::Uuid::new_v4()), + use db::schema::sled_resource::dsl; + diesel::insert_into(dsl::sled_resource) + .values(SledResource::new_for_vmm( + PropolisUuid::new_v4(), + instance, + SledUuid::new_v4(), + Resources::new( + 1, + ByteCount::from_kibibytes_u32(1).into(), + ByteCount::from_kibibytes_u32(1).into(), + ), )) .execute_async( &*datastore.pool_connection_for_tests().await.unwrap(), @@ -1050,6 +1046,20 @@ mod tests { .unwrap(); } + async fn delete_instance_reservation( + datastore: &DataStore, + instance: InstanceUuid, + ) { + use db::schema::sled_resource::dsl; + diesel::delete(dsl::sled_resource) + .filter(dsl::instance_id.eq(instance.into_untyped_uuid())) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + #[tokio::test] async fn affinity_groups_are_project_scoped() { // Setup @@ -1673,7 +1683,12 @@ mod tests { "my-instance", ) .await; - set_instance_state_running(&datastore, instance.id()).await; + + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // Cannot add the instance to the group while it's running. let err = datastore @@ -1691,13 +1706,17 @@ mod tests { assert!(matches!(err, Error::InvalidRequest { .. })); assert!( err.to_string().contains( - "Instance cannot be added to affinity group in state" + "Instance cannot be added to affinity group with reservation" ), "{err:?}" ); - // If we stop the instance, we can add it to the group. - set_instance_state_stopped(&datastore, instance.id()).await; + // If we have no reservation for the instance, we can add it to the group. + delete_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; datastore .affinity_group_member_add( &opctx, @@ -1709,8 +1728,12 @@ mod tests { .await .unwrap(); - // Now we can set the instance state to "running" once more. - set_instance_state_running(&datastore, instance.id()).await; + // Now we can reserve a sled for the instance once more. + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // We should now be able to list the new member let members = datastore @@ -1796,7 +1819,11 @@ mod tests { "my-instance", ) .await; - set_instance_state_running(&datastore, instance.id()).await; + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // Cannot add the instance to the group while it's running. let err = datastore @@ -1812,13 +1839,17 @@ mod tests { assert!(matches!(err, Error::InvalidRequest { .. })); assert!( err.to_string().contains( - "Instance cannot be added to anti-affinity group in state" + "Instance cannot be added to anti-affinity group with reservation" ), "{err:?}" ); - // If we stop the instance, we can add it to the group. - set_instance_state_stopped(&datastore, instance.id()).await; + // If we have no reservation for the instance, we can add it to the group. + delete_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; datastore .anti_affinity_group_member_add( &opctx, @@ -1830,8 +1861,12 @@ mod tests { .await .unwrap(); - // Now we can set the instance state to "running" once more. - set_instance_state_running(&datastore, instance.id()).await; + // Now we can reserve a sled for the instance once more. + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // We should now be able to list the new member let members = datastore @@ -2314,7 +2349,7 @@ mod tests { assert!( matches!(err, Error::ObjectNotFound { type_name, .. - } if type_name == ResourceType::Instance), + } if type_name == ResourceType::AffinityGroupMember), "{err:?}" ); } @@ -2474,7 +2509,7 @@ mod tests { assert!( matches!(err, Error::ObjectNotFound { type_name, .. - } if type_name == ResourceType::Instance), + } if type_name == ResourceType::AntiAffinityGroupMember), "{err:?}" ); } From 4d26262ab67d305d4c3052d1f95eef52f2cb7252 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 16:03:08 -0800 Subject: [PATCH 16/84] comment --- nexus/db-queries/src/db/datastore/affinity.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 5cd7c579105..101af1910d2 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -664,7 +664,7 @@ impl DataStore { })?; // NOTE: It may be possible to add non-stopped instances to - // affinity groups, depending on where they have already + // anti-affinity groups, depending on where they have already // been placed. However, only operating on "stopped" // instances is much easier to work with, as it does not // require any understanding of the group policy. From 6ae191011d49b1adec0aba75b7f37b088d23b456 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 16:03:59 -0800 Subject: [PATCH 17/84] clippy --- nexus/db-queries/src/db/datastore/affinity.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 101af1910d2..e55afe41eac 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -548,9 +548,7 @@ impl DataStore { // require any understanding of the group policy. if has_reservation { return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to affinity group with reservation" - ) + "Instance cannot be added to affinity group with reservation".to_string() ))); } @@ -670,9 +668,7 @@ impl DataStore { // require any understanding of the group policy. if has_reservation { return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to anti-affinity group with reservation" - ) + "Instance cannot be added to anti-affinity group with reservation".to_string() ))); } From 9daf9232b6fac3a770c4304fd7ce7df35afd17dc Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 3 Feb 2025 15:12:01 -0800 Subject: [PATCH 18/84] Converting sled_resource to sled_resource_vmm --- dev-tools/omdb/src/bin/omdb/db.rs | 5 +-- nexus/db-model/src/lib.rs | 6 +-- nexus/db-model/src/schema.rs | 5 +-- nexus/db-model/src/sled_resource_kind.rs | 19 -------- ...{sled_resource.rs => sled_resource_vmm.rs} | 20 ++++----- nexus/db-queries/src/db/datastore/affinity.rs | 16 +++---- nexus/db-queries/src/db/datastore/sled.rs | 18 ++++---- nexus/db-queries/src/db/queries/affinity.rs | 20 ++++----- .../output/lookup_affinity_sleds_query.sql | 4 +- .../lookup_anti_affinity_sleds_query.sql | 4 +- .../background/tasks/abandoned_vmm_reaper.rs | 38 +++++++--------- nexus/src/app/sagas/instance_create.rs | 3 +- nexus/src/app/sagas/instance_migrate.rs | 5 +-- nexus/src/app/sagas/instance_start.rs | 5 +-- .../app/sagas/instance_update/destroyed.rs | 3 +- nexus/src/app/sagas/instance_update/mod.rs | 10 ++--- nexus/src/app/sagas/test_helpers.rs | 18 +++----- nexus/src/app/sled.rs | 4 +- nexus/tests/integration_tests/schema.rs | 45 ------------------- schema/crdb/dbinit.sql | 33 +++++--------- 20 files changed, 87 insertions(+), 194 deletions(-) delete mode 100644 nexus/db-model/src/sled_resource_kind.rs rename nexus/db-model/src/{sled_resource.rs => sled_resource_vmm.rs} (78%) diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index c5f8a1f26c5..bc64a2c48eb 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -6141,7 +6141,7 @@ async fn cmd_db_vmm_info( &VmmInfoArgs { uuid }: &VmmInfoArgs, ) -> Result<(), anyhow::Error> { use db::schema::migration::dsl as migration_dsl; - use db::schema::sled_resource::dsl as resource_dsl; + use db::schema::sled_resource_vmm::dsl as resource_dsl; use db::schema::vmm::dsl as vmm_dsl; let vmm = vmm_dsl::vmm @@ -6184,7 +6184,6 @@ async fn cmd_db_vmm_info( let db::model::SledResource { id: _, sled_id, - kind: _, resources: db::model::Resources { hardware_threads, @@ -6206,7 +6205,7 @@ async fn cmd_db_vmm_info( println!(" {RESERVOIR:>WIDTH$}: {reservoir}"); } - let reservations = resource_dsl::sled_resource + let reservations = resource_dsl::sled_resource_vmm .filter(resource_dsl::id.eq(uuid)) .select(db::model::SledResource::as_select()) .load_async::( diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 939388174e8..b427cfa4b24 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -92,8 +92,7 @@ mod silo_user_password_hash; mod sled; mod sled_instance; mod sled_policy; -mod sled_resource; -mod sled_resource_kind; +mod sled_resource_vmm; mod sled_state; mod sled_underlay_subnet_allocation; mod snapshot; @@ -203,8 +202,7 @@ pub use silo_user_password_hash::*; pub use sled::*; pub use sled_instance::*; pub use sled_policy::to_db_sled_policy; // Do not expose DbSledPolicy -pub use sled_resource::*; -pub use sled_resource_kind::*; +pub use sled_resource_vmm::*; pub use sled_state::*; pub use sled_underlay_subnet_allocation::*; pub use snapshot::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 8881d7dd0ed..54ab96ce789 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -954,13 +954,12 @@ table! { } table! { - sled_resource (id) { + sled_resource_vmm (id) { id -> Uuid, sled_id -> Uuid, hardware_threads -> Int8, rss_ram -> Int8, reservoir_ram -> Int8, - kind -> crate::SledResourceKindEnum, instance_id -> Nullable, } } @@ -2099,7 +2098,7 @@ allow_tables_to_appear_in_same_query!( identity_provider, console_session, sled, - sled_resource, + sled_resource_vmm, support_bundle, router_route, vmm, diff --git a/nexus/db-model/src/sled_resource_kind.rs b/nexus/db-model/src/sled_resource_kind.rs deleted file mode 100644 index b9a59bdc302..00000000000 --- a/nexus/db-model/src/sled_resource_kind.rs +++ /dev/null @@ -1,19 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use super::impl_enum_type; -use serde::{Deserialize, Serialize}; - -impl_enum_type!( - #[derive(Clone, SqlType, Debug, QueryId)] - #[diesel(postgres_type(name = "sled_resource_kind", schema = "public"))] - pub struct SledResourceKindEnum; - - #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)] - #[diesel(sql_type = SledResourceKindEnum)] - pub enum SledResourceKind; - - // Enum values - Instance => b"instance" -); diff --git a/nexus/db-model/src/sled_resource.rs b/nexus/db-model/src/sled_resource_vmm.rs similarity index 78% rename from nexus/db-model/src/sled_resource.rs rename to nexus/db-model/src/sled_resource_vmm.rs index de5833aa41f..f069358087e 100644 --- a/nexus/db-model/src/sled_resource.rs +++ b/nexus/db-model/src/sled_resource_vmm.rs @@ -2,22 +2,22 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::schema::sled_resource; +use crate::schema::sled_resource_vmm; use crate::typed_uuid::DbTypedUuid; -use crate::{ByteCount, SledResourceKind, SqlU32}; -use omicron_uuid_kinds::GenericUuid; +use crate::{ByteCount, SqlU32}; use omicron_uuid_kinds::InstanceKind; use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisKind; use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledKind; use omicron_uuid_kinds::SledUuid; -use uuid::Uuid; type DbInstanceUuid = DbTypedUuid; +type DbPropolisUuid = DbTypedUuid; type DbSledUuid = DbTypedUuid; #[derive(Clone, Selectable, Queryable, Insertable, Debug)] -#[diesel(table_name = sled_resource)] +#[diesel(table_name = sled_resource_vmm)] pub struct Resources { pub hardware_threads: SqlU32, pub rss_ram: ByteCount, @@ -34,17 +34,16 @@ impl Resources { } } -/// Describes sled resource usage by services +/// Describes sled resource usage by a VMM #[derive(Clone, Selectable, Queryable, Insertable, Debug)] -#[diesel(table_name = sled_resource)] +#[diesel(table_name = sled_resource_vmm)] pub struct SledResource { - pub id: Uuid, + pub id: DbPropolisUuid, pub sled_id: DbSledUuid, #[diesel(embed)] pub resources: Resources, - pub kind: SledResourceKind, pub instance_id: Option, } @@ -56,10 +55,9 @@ impl SledResource { resources: Resources, ) -> Self { Self { - id: id.into_untyped_uuid(), + id: id.into(), instance_id: Some(instance_id.into()), sled_id: sled_id.into(), - kind: SledResourceKind::Instance, resources, } } diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index e55afe41eac..c4b074e6a46 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -476,7 +476,7 @@ impl DataStore { use db::schema::affinity_group::dsl as group_dsl; use db::schema::affinity_group_instance_membership::dsl as membership_dsl; use db::schema::instance::dsl as instance_dsl; - use db::schema::sled_resource::dsl as resource_dsl; + use db::schema::sled_resource_vmm::dsl as resource_dsl; async move { // Check that the group exists @@ -527,7 +527,7 @@ impl DataStore { })?; let has_reservation: bool = diesel::select( diesel::dsl::exists( - resource_dsl::sled_resource + resource_dsl::sled_resource_vmm .filter(resource_dsl::instance_id.eq(instance_id.into_untyped_uuid())) ) ).get_result_async(&conn) @@ -605,7 +605,7 @@ impl DataStore { use db::schema::anti_affinity_group::dsl as group_dsl; use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; use db::schema::instance::dsl as instance_dsl; - use db::schema::sled_resource::dsl as resource_dsl; + use db::schema::sled_resource_vmm::dsl as resource_dsl; async move { // Check that the group exists @@ -647,7 +647,7 @@ impl DataStore { })?; let has_reservation: bool = diesel::select( diesel::dsl::exists( - resource_dsl::sled_resource + resource_dsl::sled_resource_vmm .filter(resource_dsl::instance_id.eq(instance_id.into_untyped_uuid())) ) ).get_result_async(&conn) @@ -1023,8 +1023,8 @@ mod tests { datastore: &DataStore, instance: InstanceUuid, ) { - use db::schema::sled_resource::dsl; - diesel::insert_into(dsl::sled_resource) + use db::schema::sled_resource_vmm::dsl; + diesel::insert_into(dsl::sled_resource_vmm) .values(SledResource::new_for_vmm( PropolisUuid::new_v4(), instance, @@ -1046,8 +1046,8 @@ mod tests { datastore: &DataStore, instance: InstanceUuid, ) { - use db::schema::sled_resource::dsl; - diesel::delete(dsl::sled_resource) + use db::schema::sled_resource_vmm::dsl; + diesel::delete(dsl::sled_resource_vmm) .filter(dsl::instance_id.eq(instance.into_untyped_uuid())) .execute_async( &*datastore.pool_connection_for_tests().await.unwrap(), diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index b402cade0fd..b80ba3765e9 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -265,9 +265,9 @@ impl DataStore { let resources = resources.clone(); async move { - use db::schema::sled_resource::dsl as resource_dsl; + use db::schema::sled_resource_vmm::dsl as resource_dsl; // Check if resource ID already exists - if so, return it. - let old_resource = resource_dsl::sled_resource + let old_resource = resource_dsl::sled_resource_vmm .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) .select(SledResource::as_select()) .limit(1) @@ -320,7 +320,7 @@ impl DataStore { // for this reservation. let mut sled_targets = sled_dsl::sled .left_join( - resource_dsl::sled_resource + resource_dsl::sled_resource_vmm .on(resource_dsl::sled_id.eq(sled_dsl::id)), ) .group_by(sled_dsl::id) @@ -541,7 +541,7 @@ impl DataStore { resources, ); - diesel::insert_into(resource_dsl::sled_resource) + diesel::insert_into(resource_dsl::sled_resource_vmm) .values(resource) .returning(SledResource::as_returning()) .get_result_async(&conn) @@ -560,11 +560,11 @@ impl DataStore { pub async fn sled_reservation_delete( &self, opctx: &OpContext, - resource_id: Uuid, + vmm_id: PropolisUuid, ) -> DeleteResult { - use db::schema::sled_resource::dsl as resource_dsl; - diesel::delete(resource_dsl::sled_resource) - .filter(resource_dsl::id.eq(resource_id)) + use db::schema::sled_resource_vmm::dsl as resource_dsl; + diesel::delete(resource_dsl::sled_resource_vmm) + .filter(resource_dsl::id.eq(vmm_id.into_untyped_uuid())) .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -1371,7 +1371,7 @@ pub(in crate::db::datastore) mod test { ); datastore - .sled_reservation_delete(&opctx, resource.id) + .sled_reservation_delete(&opctx, resource.id.into()) .await .unwrap(); } diff --git a/nexus/db-queries/src/db/queries/affinity.rs b/nexus/db-queries/src/db/queries/affinity.rs index 6da857669da..0383e0c04a0 100644 --- a/nexus/db-queries/src/db/queries/affinity.rs +++ b/nexus/db-queries/src/db/queries/affinity.rs @@ -40,10 +40,9 @@ pub fn lookup_anti_affinity_sleds_query( ) SELECT DISTINCT policy,sled_id FROM other_instances_by_policy - JOIN sled_resource + JOIN sled_resource_vmm ON - sled_resource.instance_id = other_instances_by_policy.instance_id AND - sled_resource.kind = 'instance'") + sled_resource_vmm.instance_id = other_instances_by_policy.instance_id") .bind::(instance_id.into_untyped_uuid()) .bind::(instance_id.into_untyped_uuid()) .query() @@ -87,10 +86,9 @@ pub fn lookup_affinity_sleds_query( ) SELECT DISTINCT policy,sled_id FROM other_instances_by_policy - JOIN sled_resource + JOIN sled_resource_vmm ON - sled_resource.instance_id = other_instances_by_policy.instance_id AND - sled_resource.kind = 'instance'", + sled_resource_vmm.instance_id = other_instances_by_policy.instance_id", ) .bind::(instance_id.into_untyped_uuid()) .bind::(instance_id.into_untyped_uuid()) @@ -266,7 +264,7 @@ mod test { Ok(()) } - async fn make_instance_sled_resource( + async fn make_sled_resource_vmm( sled_id: SledUuid, instance_id: InstanceUuid, conn: &async_bb8_diesel::Connection, @@ -285,8 +283,8 @@ mod test { ), ), ); - use crate::db::schema::sled_resource::dsl; - diesel::insert_into(dsl::sled_resource) + use crate::db::schema::sled_resource_vmm::dsl; + diesel::insert_into(dsl::sled_resource_vmm) .values(resource) .execute_async(conn) .await @@ -343,7 +341,7 @@ mod test { .await .unwrap(); - make_instance_sled_resource(sled_id, other_instance_id, &conn) + make_sled_resource_vmm(sled_id, other_instance_id, &conn) .await .unwrap(); @@ -499,7 +497,7 @@ mod test { .await .unwrap(); - make_instance_sled_resource(sled_id, other_instance_id, &conn) + make_sled_resource_vmm(sled_id, other_instance_id, &conn) .await .unwrap(); diff --git a/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql index 1435a0b8f62..b03bbc45fcf 100644 --- a/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql +++ b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql @@ -25,6 +25,4 @@ SELECT DISTINCT policy, sled_id FROM other_instances_by_policy - JOIN sled_resource ON - sled_resource.instance_id = other_instances_by_policy.instance_id - AND sled_resource.kind = 'instance' + JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_instances_by_policy.instance_id diff --git a/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql index d5e7cd66d86..ec81dfbcf14 100644 --- a/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql +++ b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql @@ -27,6 +27,4 @@ SELECT DISTINCT policy, sled_id FROM other_instances_by_policy - JOIN sled_resource ON - sled_resource.instance_id = other_instances_by_policy.instance_id - AND sled_resource.kind = 'instance' + JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_instances_by_policy.instance_id diff --git a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs index 1853a82fb19..164e780f8f7 100644 --- a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs +++ b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs @@ -100,11 +100,7 @@ impl AbandonedVmmReaper { let vmm_id = PropolisUuid::from_untyped_uuid(vmm.id); slog::trace!(opctx.log, "Deleting abandoned VMM"; "vmm" => %vmm_id); // Attempt to remove the abandoned VMM's sled resource reservation. - match self - .datastore - .sled_reservation_delete(opctx, vmm_id.into_untyped_uuid()) - .await - { + match self.datastore.sled_reservation_delete(opctx, vmm_id).await { Ok(_) => { slog::trace!( opctx.log, @@ -283,7 +279,7 @@ mod tests { ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, }; - use nexus_db_queries::db::schema::sled_resource::dsl as sled_resource_dsl; + use nexus_db_queries::db::schema::sled_resource_vmm::dsl as sled_resource_vmm_dsl; use nexus_db_queries::db::schema::vmm::dsl as vmm_dsl; let conn = datastore.pool_connection_for_tests().await.unwrap(); @@ -302,18 +298,19 @@ mod tests { "VMM record should have been deleted" ); - let fetched_sled_resource = sled_resource_dsl::sled_resource - .filter( - sled_resource_dsl::id - .eq(self.destroyed_vmm_id.into_untyped_uuid()), - ) - .select(SledResource::as_select()) - .first_async::(&*conn) - .await - .optional() - .expect("sled resource query should succeed"); + let fetched_sled_resource_vmm = + sled_resource_vmm_dsl::sled_resource_vmm + .filter( + sled_resource_vmm_dsl::id + .eq(self.destroyed_vmm_id.into_untyped_uuid()), + ) + .select(SledResource::as_select()) + .first_async::(&*conn) + .await + .optional() + .expect("sled resource query should succeed"); assert!( - dbg!(fetched_sled_resource).is_none(), + dbg!(fetched_sled_resource_vmm).is_none(), "sled resource record should have been deleted" ); } @@ -394,7 +391,7 @@ mod tests { } #[nexus_test(server = crate::Server)] - async fn sled_resource_already_deleted( + async fn sled_resource_vmm_already_deleted( cptestctx: &ControlPlaneTestContext, ) { let nexus = &cptestctx.server.server_context().nexus; @@ -422,10 +419,7 @@ mod tests { assert!(!abandoned_vmms.is_empty()); datastore - .sled_reservation_delete( - &opctx, - fixture.destroyed_vmm_id.into_untyped_uuid(), - ) + .sled_reservation_delete(&opctx, fixture.destroyed_vmm_id) .await .expect( "simulate another nexus marking the sled reservation deleted", diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index d9aae35596f..6a85cc11df1 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -1320,8 +1320,7 @@ pub mod test { assert!(no_network_interface_records_exist(datastore).await); assert!(no_external_ip_records_exist(datastore).await); assert!( - test_helpers::no_sled_resource_instance_records_exist(cptestctx) - .await + test_helpers::no_sled_resource_vmm_records_exist(cptestctx).await ); assert!( test_helpers::no_virtual_provisioning_resource_records_exist( diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 465105a813f..191c8f42185 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -203,10 +203,7 @@ async fn sim_release_sled_resources( let osagactx = sagactx.user_data(); let propolis_id = sagactx.lookup::("dst_propolis_id")?; - osagactx - .nexus() - .delete_sled_reservation(propolis_id.into_untyped_uuid()) - .await?; + osagactx.nexus().delete_sled_reservation(propolis_id).await?; Ok(()) } diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index e6af6453291..97211f5cccd 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -180,10 +180,7 @@ async fn sis_alloc_server_undo( let osagactx = sagactx.user_data(); let propolis_id = sagactx.lookup::("propolis_id")?; - osagactx - .nexus() - .delete_sled_reservation(propolis_id.into_untyped_uuid()) - .await?; + osagactx.nexus().delete_sled_reservation(propolis_id).await?; Ok(()) } diff --git a/nexus/src/app/sagas/instance_update/destroyed.rs b/nexus/src/app/sagas/instance_update/destroyed.rs index 243f952c8bf..9e7e2b63ce2 100644 --- a/nexus/src/app/sagas/instance_update/destroyed.rs +++ b/nexus/src/app/sagas/instance_update/destroyed.rs @@ -9,7 +9,6 @@ use super::{ use crate::app::sagas::ActionError; use nexus_db_queries::authn; use omicron_common::api::external::Error; -use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use serde::{Deserialize, Serialize}; @@ -89,7 +88,7 @@ async fn siu_destroyed_release_sled_resources( osagactx .datastore() - .sled_reservation_delete(&opctx, vmm_id.into_untyped_uuid()) + .sled_reservation_delete(&opctx, vmm_id) .await .or_else(|err| { // Necessary for idempotency diff --git a/nexus/src/app/sagas/instance_update/mod.rs b/nexus/src/app/sagas/instance_update/mod.rs index 206ecb85afe..3b1820865ad 100644 --- a/nexus/src/app/sagas/instance_update/mod.rs +++ b/nexus/src/app/sagas/instance_update/mod.rs @@ -1876,8 +1876,7 @@ mod test { virtual provisioning records if none exist!", ); assert!( - !test_helpers::no_sled_resource_instance_records_exist(cptestctx) - .await, + !test_helpers::no_sled_resource_vmm_records_exist(cptestctx).await, "we can't assert that a destroyed VMM instance update deallocates \ sled resource records if none exist!" ); @@ -1936,8 +1935,7 @@ mod test { ); assert!(test_helpers::no_virtual_provisioning_collection_records_using_instances(cptestctx).await); assert!( - test_helpers::no_sled_resource_instance_records_exist(cptestctx) - .await + test_helpers::no_sled_resource_vmm_records_exist(cptestctx).await ); } @@ -2818,7 +2816,7 @@ mod test { &self, cptestctx: &ControlPlaneTestContext, ) -> bool { - test_helpers::sled_resources_exist_for_vmm( + test_helpers::sled_resource_vmms_exist_for_vmm( cptestctx, PropolisUuid::from_untyped_uuid(self.src_vmm_id()), ) @@ -2829,7 +2827,7 @@ mod test { &self, cptestctx: &ControlPlaneTestContext, ) -> bool { - test_helpers::sled_resources_exist_for_vmm( + test_helpers::sled_resource_vmms_exist_for_vmm( cptestctx, PropolisUuid::from_untyped_uuid(self.target_vmm_id()), ) diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index 19d11bde725..a13cd286204 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -462,18 +462,17 @@ pub async fn count_virtual_provisioning_collection_records_using_instances( .unwrap() } -pub async fn no_sled_resource_instance_records_exist( +pub async fn no_sled_resource_vmm_records_exist( cptestctx: &ControlPlaneTestContext, ) -> bool { use nexus_db_queries::db::model::SledResource; - use nexus_db_queries::db::model::SledResourceKind; - use nexus_db_queries::db::schema::sled_resource::dsl; + use nexus_db_queries::db::schema::sled_resource_vmm::dsl; let datastore = cptestctx.server.server_context().nexus.datastore(); let conn = datastore.pool_connection_for_tests().await.unwrap(); datastore - .transaction_retry_wrapper("no_sled_resource_instance_records_exist") + .transaction_retry_wrapper("no_sled_resource_vmm_records_exist") .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, @@ -481,8 +480,7 @@ pub async fn no_sled_resource_instance_records_exist( .await .unwrap(); - Ok(dsl::sled_resource - .filter(dsl::kind.eq(SledResourceKind::Instance)) + Ok(dsl::sled_resource_vmm .select(SledResource::as_select()) .get_results_async::(&conn) .await @@ -493,19 +491,17 @@ pub async fn no_sled_resource_instance_records_exist( .unwrap() } -pub async fn sled_resources_exist_for_vmm( +pub async fn sled_resource_vmms_exist_for_vmm( cptestctx: &ControlPlaneTestContext, vmm_id: PropolisUuid, ) -> bool { use nexus_db_queries::db::model::SledResource; - use nexus_db_queries::db::model::SledResourceKind; - use nexus_db_queries::db::schema::sled_resource::dsl; + use nexus_db_queries::db::schema::sled_resource_vmm::dsl; let datastore = cptestctx.server.server_context().nexus.datastore(); let conn = datastore.pool_connection_for_tests().await.unwrap(); - let results = dsl::sled_resource - .filter(dsl::kind.eq(SledResourceKind::Instance)) + let results = dsl::sled_resource_vmm .filter(dsl::id.eq(vmm_id.into_untyped_uuid())) .select(SledResource::as_select()) .load_async(&*conn) diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index ffa507ffb8e..27380b53956 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -184,10 +184,10 @@ impl super::Nexus { pub(crate) async fn delete_sled_reservation( &self, - resource_id: Uuid, + vmm_id: PropolisUuid, ) -> Result<(), Error> { self.db_datastore - .sled_reservation_delete(&self.opctx_alloc, resource_id) + .sled_reservation_delete(&self.opctx_alloc, vmm_id) .await } diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index cbc041f741a..0e0d4832718 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -17,8 +17,6 @@ use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; -use omicron_uuid_kinds::InstanceUuid; -use omicron_uuid_kinds::SledUuid; use pretty_assertions::{assert_eq, assert_ne}; use similar_asserts; use slog::Logger; @@ -945,16 +943,6 @@ async fn dbinit_equals_sum_of_all_up() { ); } - // Create a connection pool after we apply the first schema version but - // before applying the rest, and grab a connection from that pool. We'll use - // it for an extra check later. - let pool = nexus_db_queries::db::Pool::new_single_host( - log, - &nexus_db_queries::db::Config { url: crdb.pg_config().clone() }, - ); - let conn_from_pool = - pool.claim().await.expect("failed to get pooled connection"); - // Go from the second version to the latest version. for version in all_versions.iter_versions().skip(1) { apply_update(log, &crdb, version, 1).await; @@ -972,39 +960,6 @@ async fn dbinit_equals_sum_of_all_up() { let observed_schema = InformationSchema::new(&crdb).await; let observed_data = observed_schema.query_all_tables(log, &crdb).await; - // Using the connection we got from the connection pool prior to applying - // the schema migrations, attempt to insert a sled resource. This involves - // the `sled_resource_kind` enum, whose OID was changed by the schema - // migration in version 53.0.0 (by virtue of the enum being dropped and - // added back with a different set of variants). If the diesel OID cache was - // populated when we acquired the connection from the pool, this will fail - // with a `type with ID $NUM does not exist` error. - { - use async_bb8_diesel::AsyncRunQueryDsl; - use nexus_db_model::schema::sled_resource::dsl; - use nexus_db_model::Resources; - use nexus_db_model::SledResource; - use nexus_db_model::SledResourceKind; - - diesel::insert_into(dsl::sled_resource) - .values(SledResource { - id: Uuid::new_v4(), - instance_id: Some(InstanceUuid::new_v4().into()), - sled_id: SledUuid::new_v4().into(), - kind: SledResourceKind::Instance, - resources: Resources { - hardware_threads: 8_u32.into(), - rss_ram: 1024_i64.try_into().unwrap(), - reservoir_ram: 1024_i64.try_into().unwrap(), - }, - }) - .execute_async(&*conn_from_pool) - .await - .expect("failed to insert - did we poison the OID cache?"); - } - std::mem::drop(conn_from_pool); - pool.terminate().await; - std::mem::drop(pool); db.terminate().await; // Create a new DB with data populated from dbinit.sql for comparison diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index b8e110788d6..549121b7861 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -216,37 +216,26 @@ CREATE INDEX IF NOT EXISTS lookup_sled_by_policy_and_state ON omicron.public.sle sled_state ); -CREATE TYPE IF NOT EXISTS omicron.public.sled_resource_kind AS ENUM ( - -- omicron.public.vmm ; this is called "instance" for historical reasons. - 'instance' - -- We expect to other resource kinds here in the future; e.g., to track - -- resources used by control plane services. For now, we only track - -- instances. -); - --- Accounting for programs using resources on a sled -CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( - -- Should match the UUID of the corresponding resource +-- Accounting for instances using resources on a sled +CREATE TABLE IF NOT EXISTS omicron.public.sled_resource_vmm ( + -- Should match the UUID of the corresponding VMM id UUID PRIMARY KEY, -- The sled where resources are being consumed sled_id UUID NOT NULL, - -- The maximum number of hardware threads usable by this resource + -- The maximum number of hardware threads usable by this VMM hardware_threads INT8 NOT NULL, - -- The maximum amount of RSS RAM provisioned to this resource + -- The maximum amount of RSS RAM provisioned to this VMM rss_ram INT8 NOT NULL, - -- The maximum amount of Reservoir RAM provisioned to this resource + -- The maximum amount of Reservoir RAM provisioned to this VMM reservoir_ram INT8 NOT NULL, - -- Identifies the type of the resource - kind omicron.public.sled_resource_kind NOT NULL, - - -- The UUID of an instance, if this resource belongs to an instance. + -- The UUID of the instance to which this VMM belongs. -- - -- This should eventually become NOT NULL for all instances, but is + -- This should eventually become NOT NULL for all VMMs, but is -- still nullable for backwards compatibility purposes. Specifically, -- the "instance start" saga can create rows in this table before creating -- rows for "omicron.public.vmm", which we would use for back-filling. @@ -256,14 +245,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( instance_id UUID ); --- Allow looking up all resources which reside on a sled -CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled_resource ( +-- Allow looking up all VMM resources which reside on a sled +CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled_resource_vmm ( sled_id, id ); -- Allow looking up all resources by instance -CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource_vmm ( instance_id ); From 1b39c511a124f6bbfbdb92ec80c7ee4c3b8b588e Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 3 Feb 2025 15:25:40 -0800 Subject: [PATCH 19/84] schema, more renaming --- dev-tools/omdb/src/bin/omdb/db.rs | 8 ++++---- nexus/db-model/src/schema_versions.rs | 3 ++- nexus/db-model/src/sled_resource_vmm.rs | 6 +++--- nexus/db-queries/src/db/datastore/affinity.rs | 4 ++-- nexus/db-queries/src/db/datastore/sled.rs | 20 ++++++++++--------- nexus/db-queries/src/db/queries/affinity.rs | 2 +- .../background/tasks/abandoned_vmm_reaper.rs | 6 +++--- nexus/src/app/sagas/instance_common.rs | 4 ++-- nexus/src/app/sagas/test_helpers.rs | 10 +++++----- nexus/src/app/sled.rs | 2 +- schema/crdb/dbinit.sql | 8 ++++---- schema/crdb/sled-resource-for-vmm/up01.sql | 1 + schema/crdb/sled-resource-for-vmm/up02.sql | 1 + schema/crdb/sled-resource-for-vmm/up03.sql | 1 + schema/crdb/sled-resource-for-vmm/up04.sql | 1 + schema/crdb/sled-resource-for-vmm/up05.sql | 1 + 16 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 schema/crdb/sled-resource-for-vmm/up01.sql create mode 100644 schema/crdb/sled-resource-for-vmm/up02.sql create mode 100644 schema/crdb/sled-resource-for-vmm/up03.sql create mode 100644 schema/crdb/sled-resource-for-vmm/up04.sql create mode 100644 schema/crdb/sled-resource-for-vmm/up05.sql diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index bc64a2c48eb..e497cefb579 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -6177,11 +6177,11 @@ async fn cmd_db_vmm_info( ); fn prettyprint_reservation( - resource: db::model::SledResource, + resource: db::model::SledResourceVmm, include_sled_id: bool, ) { use db::model::ByteCount; - let db::model::SledResource { + let db::model::SledResourceVmm { id: _, sled_id, resources: @@ -6207,8 +6207,8 @@ async fn cmd_db_vmm_info( let reservations = resource_dsl::sled_resource_vmm .filter(resource_dsl::id.eq(uuid)) - .select(db::model::SledResource::as_select()) - .load_async::( + .select(db::model::SledResourceVmm::as_select()) + .load_async::( &*datastore.pool_connection_for_tests().await?, ) .await diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 852de12f4c5..8b232709316 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(123, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(124, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(124, "sled-resource-for-vmm"), KnownVersion::new(123, "affinity"), KnownVersion::new(122, "tuf-artifact-replication"), KnownVersion::new(121, "dataset-to-crucible-dataset"), diff --git a/nexus/db-model/src/sled_resource_vmm.rs b/nexus/db-model/src/sled_resource_vmm.rs index f069358087e..1849f49b871 100644 --- a/nexus/db-model/src/sled_resource_vmm.rs +++ b/nexus/db-model/src/sled_resource_vmm.rs @@ -37,7 +37,7 @@ impl Resources { /// Describes sled resource usage by a VMM #[derive(Clone, Selectable, Queryable, Insertable, Debug)] #[diesel(table_name = sled_resource_vmm)] -pub struct SledResource { +pub struct SledResourceVmm { pub id: DbPropolisUuid, pub sled_id: DbSledUuid, @@ -47,8 +47,8 @@ pub struct SledResource { pub instance_id: Option, } -impl SledResource { - pub fn new_for_vmm( +impl SledResourceVmm { + pub fn new( id: PropolisUuid, instance_id: InstanceUuid, sled_id: SledUuid, diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index c4b074e6a46..3fe3660e7dd 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -878,7 +878,7 @@ mod tests { use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::Instance; use nexus_db_model::Resources; - use nexus_db_model::SledResource; + use nexus_db_model::SledResourceVmm; use nexus_types::external_api::params; use omicron_common::api::external::{ self, ByteCount, DataPageParams, IdentityMetadataCreateParams, @@ -1025,7 +1025,7 @@ mod tests { ) { use db::schema::sled_resource_vmm::dsl; diesel::insert_into(dsl::sled_resource_vmm) - .values(SledResource::new_for_vmm( + .values(SledResourceVmm::new( PropolisUuid::new_v4(), instance, SledUuid::new_v4(), diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index b80ba3765e9..c5e073b5304 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -15,7 +15,7 @@ use crate::db::error::ErrorHandler; use crate::db::model::to_db_sled_policy; use crate::db::model::AffinityPolicy; use crate::db::model::Sled; -use crate::db::model::SledResource; +use crate::db::model::SledResourceVmm; use crate::db::model::SledState; use crate::db::model::SledUpdate; use crate::db::pagination::paginated; @@ -217,7 +217,7 @@ impl DataStore { propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, - ) -> CreateResult { + ) -> CreateResult { self.sled_reservation_create_inner( opctx, instance_id, @@ -253,7 +253,8 @@ impl DataStore { propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, - ) -> Result { + ) -> Result + { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; @@ -269,7 +270,7 @@ impl DataStore { // Check if resource ID already exists - if so, return it. let old_resource = resource_dsl::sled_resource_vmm .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) - .select(SledResource::as_select()) + .select(SledResourceVmm::as_select()) .limit(1) .load_async(&conn) .await?; @@ -532,9 +533,9 @@ impl DataStore { } }; - // Create a SledResource record, associate it with the target + // Create a SledResourceVmm record, associate it with the target // sled. - let resource = SledResource::new_for_vmm( + let resource = SledResourceVmm::new( propolis_id, instance_id, SledUuid::from_untyped_uuid(sled_target), @@ -543,7 +544,7 @@ impl DataStore { diesel::insert_into(resource_dsl::sled_resource_vmm) .values(resource) - .returning(SledResource::as_returning()) + .returning(SledResourceVmm::as_returning()) .get_result_async(&conn) .await } @@ -1647,7 +1648,7 @@ pub(in crate::db::datastore) mod test { opctx: &OpContext, datastore: &DataStore, all_groups: &AllGroups, - ) -> Result + ) -> Result { self.add_to_groups(&datastore, &all_groups).await; create_instance_reservation(&datastore, &opctx, self).await @@ -1689,7 +1690,8 @@ pub(in crate::db::datastore) mod test { db: &DataStore, opctx: &OpContext, instance: &Instance, - ) -> Result { + ) -> Result + { // Pick a specific sled, if requested let constraints = db::model::SledReservationConstraintBuilder::new(); let constraints = if let Some(sled_target) = instance.force_onto_sled { diff --git a/nexus/db-queries/src/db/queries/affinity.rs b/nexus/db-queries/src/db/queries/affinity.rs index 0383e0c04a0..03a28e01417 100644 --- a/nexus/db-queries/src/db/queries/affinity.rs +++ b/nexus/db-queries/src/db/queries/affinity.rs @@ -269,7 +269,7 @@ mod test { instance_id: InstanceUuid, conn: &async_bb8_diesel::Connection, ) -> anyhow::Result<()> { - let resource = model::SledResource::new_for_vmm( + let resource = model::SledResourceVmm::new( PropolisUuid::new_v4(), instance_id, sled_id, diff --git a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs index 164e780f8f7..cb21ff8a536 100644 --- a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs +++ b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs @@ -196,7 +196,7 @@ mod tests { use nexus_db_model::ByteCount; use nexus_db_model::Generation; use nexus_db_model::Resources; - use nexus_db_model::SledResource; + use nexus_db_model::SledResourceVmm; use nexus_db_model::Vmm; use nexus_db_model::VmmRuntimeState; use nexus_db_model::VmmState; @@ -304,8 +304,8 @@ mod tests { sled_resource_vmm_dsl::id .eq(self.destroyed_vmm_id.into_untyped_uuid()), ) - .select(SledResource::as_select()) - .first_async::(&*conn) + .select(SledResourceVmm::as_select()) + .first_async::(&*conn) .await .optional() .expect("sled resource query should succeed"); diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index 22134e8f6d1..6c4fcd82826 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -9,7 +9,7 @@ use std::net::{IpAddr, Ipv6Addr}; use crate::Nexus; use nexus_db_model::{ ByteCount, ExternalIp, InstanceState, IpAttachState, Ipv4NatEntry, - SledReservationConstraints, SledResource, VmmState, + SledReservationConstraints, SledResourceVmm, VmmState, }; use nexus_db_queries::authz; use nexus_db_queries::db::lookup::LookupPath; @@ -43,7 +43,7 @@ pub async fn reserve_vmm_resources( ncpus: u32, guest_memory: ByteCount, constraints: SledReservationConstraints, -) -> Result { +) -> Result { // ALLOCATION POLICY // // NOTE: This policy can - and should! - be changed. diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index a13cd286204..b5cef11cceb 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -465,7 +465,7 @@ pub async fn count_virtual_provisioning_collection_records_using_instances( pub async fn no_sled_resource_vmm_records_exist( cptestctx: &ControlPlaneTestContext, ) -> bool { - use nexus_db_queries::db::model::SledResource; + use nexus_db_queries::db::model::SledResourceVmm; use nexus_db_queries::db::schema::sled_resource_vmm::dsl; let datastore = cptestctx.server.server_context().nexus.datastore(); @@ -481,8 +481,8 @@ pub async fn no_sled_resource_vmm_records_exist( .unwrap(); Ok(dsl::sled_resource_vmm - .select(SledResource::as_select()) - .get_results_async::(&conn) + .select(SledResourceVmm::as_select()) + .get_results_async::(&conn) .await .unwrap() .is_empty()) @@ -495,7 +495,7 @@ pub async fn sled_resource_vmms_exist_for_vmm( cptestctx: &ControlPlaneTestContext, vmm_id: PropolisUuid, ) -> bool { - use nexus_db_queries::db::model::SledResource; + use nexus_db_queries::db::model::SledResourceVmm; use nexus_db_queries::db::schema::sled_resource_vmm::dsl; let datastore = cptestctx.server.server_context().nexus.datastore(); @@ -503,7 +503,7 @@ pub async fn sled_resource_vmms_exist_for_vmm( let results = dsl::sled_resource_vmm .filter(dsl::id.eq(vmm_id.into_untyped_uuid())) - .select(SledResource::as_select()) + .select(SledResourceVmm::as_select()) .load_async(&*conn) .await .unwrap(); diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index 27380b53956..796983ebf38 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -170,7 +170,7 @@ impl super::Nexus { propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, - ) -> Result { + ) -> Result { self.db_datastore .sled_reservation_create( &self.opctx_alloc, diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 549121b7861..62adfecc86f 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -216,7 +216,7 @@ CREATE INDEX IF NOT EXISTS lookup_sled_by_policy_and_state ON omicron.public.sle sled_state ); --- Accounting for instances using resources on a sled +-- Accounting for VMMs using resources on a sled CREATE TABLE IF NOT EXISTS omicron.public.sled_resource_vmm ( -- Should match the UUID of the corresponding VMM id UUID PRIMARY KEY, @@ -246,13 +246,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource_vmm ( ); -- Allow looking up all VMM resources which reside on a sled -CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled_resource_vmm ( +CREATE UNIQUE INDEX IF NOT EXISTS lookup_vmm_resource_by_sled ON omicron.public.sled_resource_vmm ( sled_id, id ); -- Allow looking up all resources by instance -CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource_vmm ( +CREATE INDEX IF NOT EXISTS lookup_vmm_resource_by_instance ON omicron.public.sled_resource_vmm ( instance_id ); @@ -4902,7 +4902,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '123.0.0', NULL) + (TRUE, NOW(), NOW(), '124.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/sled-resource-for-vmm/up01.sql b/schema/crdb/sled-resource-for-vmm/up01.sql new file mode 100644 index 00000000000..49a7994622a --- /dev/null +++ b/schema/crdb/sled-resource-for-vmm/up01.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.sled_resource DROP COLUMN IF EXISTS kind; diff --git a/schema/crdb/sled-resource-for-vmm/up02.sql b/schema/crdb/sled-resource-for-vmm/up02.sql new file mode 100644 index 00000000000..f8680b4460b --- /dev/null +++ b/schema/crdb/sled-resource-for-vmm/up02.sql @@ -0,0 +1 @@ +DROP TYPE IF EXISTS sled_resource_kind; diff --git a/schema/crdb/sled-resource-for-vmm/up03.sql b/schema/crdb/sled-resource-for-vmm/up03.sql new file mode 100644 index 00000000000..baf33aeda45 --- /dev/null +++ b/schema/crdb/sled-resource-for-vmm/up03.sql @@ -0,0 +1 @@ +ALTER TABLE IF EXISTS omicron.public.sled_resource RENAME TO omicron.public.sled_resource_vmm; diff --git a/schema/crdb/sled-resource-for-vmm/up04.sql b/schema/crdb/sled-resource-for-vmm/up04.sql new file mode 100644 index 00000000000..2ca16609854 --- /dev/null +++ b/schema/crdb/sled-resource-for-vmm/up04.sql @@ -0,0 +1 @@ +ALTER INDEX IF EXISTS lookup_resource_by_sled RENAME TO lookup_vmm_resource_by_sled; diff --git a/schema/crdb/sled-resource-for-vmm/up05.sql b/schema/crdb/sled-resource-for-vmm/up05.sql new file mode 100644 index 00000000000..cfbb7ce8fba --- /dev/null +++ b/schema/crdb/sled-resource-for-vmm/up05.sql @@ -0,0 +1 @@ +ALTER INDEX IF EXISTS lookup_resource_by_vmm RENAME TO lookup_vmm_resource_by_instance; From a07c46e17e48b0bca2896be593945f4403bc458c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 4 Feb 2025 15:48:01 -0800 Subject: [PATCH 20/84] review feedback --- nexus/db-queries/Cargo.toml | 1 + nexus/db-queries/src/db/datastore/sled.rs | 437 +++++++++++--------- nexus/db-queries/src/db/queries/affinity.rs | 15 +- 3 files changed, 250 insertions(+), 203 deletions(-) diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index db2b70488d7..f51629f63cd 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -25,6 +25,7 @@ futures.workspace = true internal-dns-resolver.workspace = true internal-dns-types.workspace = true ipnetwork.workspace = true +itertools.workspace = true macaddr.workspace = true once_cell.workspace = true oxnet.workspace = true diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index b402cade0fd..bb61d8eebac 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -29,6 +29,8 @@ use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use itertools::Either; +use itertools::Itertools; use nexus_db_model::ApplySledFilterExt; use nexus_types::deployment::SledFilter; use nexus_types::external_api::views::SledPolicy; @@ -46,6 +48,7 @@ use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; +use slog::Logger; use std::collections::HashSet; use std::fmt; use strum::IntoEnumIterator; @@ -54,16 +57,62 @@ use uuid::Uuid; #[derive(Debug, thiserror::Error)] enum SledReservationError { - #[error("No reservation could be found")] + #[error( + "Could not find any valid sled on which this instance can be placed" + )] NotFound, - #[error("More than one sled was found with 'affinity = required'")] + #[error("This instance belongs to an affinity group that requires it be placed \ + on more than one sled. Instances can only placed on a single sled, so \ + this is impossible to satisfy - Consider stopping other instances in \ + the affinity group.")] TooManyAffinityConstraints, - #[error("A sled that is required is also considered banned by anti-affinity rules")] + #[error( + "This instance belongs to an affinity group that requires it to be \ + placed on a sled, but also belongs to an anti-affinity group that \ + prevents it from being placed on that sled. These constraints are \ + contradictory: Consider stopping instances in those \ + affinity/anti-affinity groups, or changing group membership." + )] ConflictingAntiAndAffinityConstraints, - #[error("A sled required by an affinity group is not a valid target for other reasons")] + #[error("This instance must be placed on a specific sled due to affinity \ + rules, but that placement is invalid for some other reason (e.g., \ + the instance would not fit, or the sled sharing an instance in the \ + affinity group is being expunged). Consider stopping other \ + instances in the affinity group, or changing affinity group \ + membership.")] RequiredAffinitySledNotValid, } +impl From for external::Error { + fn from(err: SledReservationError) -> Self { + let msg = format!("Failed to place instance: {err}"); + match err { + // "NotFound" can be resolved by adding more capacity + SledReservationError::NotFound + // "RequiredAffinitySledNotValid" is *usually* the result of a + // single sled filling up with several members of an Affinity group. + // + // It is also possible briefly when a sled with affinity groups is + // being expunged with running VMMs, but "insufficient capacity" is + // the much more common case. + // + // (Disambiguating these cases would require additional database + // queries, hence why this isn't being done right now) + | SledReservationError::RequiredAffinitySledNotValid => { + external::Error::insufficient_capacity(&msg, &msg) + }, + // The following cases are constraint violations due to excessive + // affinity/anti-affinity groups -- even if additional capacity is + // added, they won't be fixed. Return a 400 error to signify to the + // caller that they're responsible for fixing these constraints. + SledReservationError::TooManyAffinityConstraints + | SledReservationError::ConflictingAntiAndAffinityConstraints => { + external::Error::invalid_request(&msg) + }, + } + } +} + #[derive(Debug, thiserror::Error)] enum SledReservationTransactionError { #[error(transparent)] @@ -74,6 +123,166 @@ enum SledReservationTransactionError { Reservation(#[from] SledReservationError), } +// Chooses a sled for reservation with the supplied constraints. +// +// - "targets": All possible sleds which might be selected. +// - "anti_affinity_sleds": All sleds which are anti-affine to +// our requested reservation. +// - "affinity_sleds": All sleds which are affine to our requested +// reservation. +// +// We use the following logic to calculate a desirable sled, given a possible +// set of "targets", and the information from affinity groups. +// +// # Rules vs Preferences +// +// Due to the flavors of "affinity policy", it's possible to bucket affinity +// choices into two categories: "rules" and "preferences". "rules" are affinity +// dispositions for or against sled placement that must be followed, and +// "preferences" are affinity dispositions that should be followed for sled +// selection, in order of "most preferential" to "least preferential". +// +// As example of a "rule" is "an anti-affinity group exists, containing a +// target sled, with affinity_policy = 'fail'". +// +// An example of a "preference" is "an anti-affinity group exists, containing a +// target sled, but the policy is 'allow'." We don't want to use it as a target, +// but we will if there are no other choices. +// +// We apply rules before preferences to ensure they are always respected. +// Furthermore, the evaluation of preferences is a target-seeking operation, +// which identifies the distinct sets of targets, and searches them in +// decreasing preference order. +// +// # Logic +// +// ## Background: Notation +// +// We use the following symbols for sets below: +// - ∩: Intersection of two sets (A ∩ B is "everything that exists in A and +// also exists in B"). +// - ∖: difference of two sets (A ∖ B is "everything that exists in A that does +// not exist in B). +// +// We also use the following notation for brevity: +// - AA,P=Fail: All sleds containing instances that are part of an anti-affinity +// group with policy = 'fail'. +// - AA,P=Allow: Same as above, but with policy = 'allow'. +// - A,P=Fail: All sleds containing instances that are part of an affinity group +// with policy = 'fail'. +// - A,P=Allow: Same as above, but with policy = 'allow'. +// +// ## Affinity: Apply Rules +// +// - Targets := All viable sleds for instance placement +// - Banned := AA,P=Fail +// - Required := A,P=Fail +// - if Required.len() > 1: Fail (too many constraints). +// - if Required.len() == 1... +// - ... if the entry exists in the "Banned" set: Fail +// (contradicting constraints 'Banned' + 'Required') +// - ... if the entry does not exist in "Targets": Fail +// ('Required' constraint not satisfiable) +// - ... if the entry does not exist in "Banned": Use it. +// +// If we have not yet picked a target, we can filter the set of targets to +// ignore "banned" sleds, and then apply preferences. +// +// - Targets := Targets ∖ Banned +// +// ## Affinity: Apply Preferences +// +// - Preferred := Targets ∩ A,P=Allow +// - Unpreferred := Targets ∩ AA,P=Allow +// - Both := Preferred ∩ Unpreferred +// - Preferred := Preferred ∖ Both +// - Unpreferred := Unpreferred ∖ Both +// - If Preferred isn't empty, pick a target from it. +// - Targets := Targets \ Unpreferred +// - If Targets isn't empty, pick a target from it. +// - If Unpreferred isn't empty, pick a target from it. +// - Fail, no targets are available. +fn pick_sled_reservation_target( + log: &Logger, + mut targets: HashSet, + anti_affinity_sleds: Vec<(AffinityPolicy, Uuid)>, + affinity_sleds: Vec<(AffinityPolicy, Uuid)>, +) -> Result { + let (banned, mut unpreferred): (HashSet<_>, HashSet<_>) = + anti_affinity_sleds.into_iter().partition_map(|(policy, id)| { + let id = SledUuid::from_untyped_uuid(id); + match policy { + AffinityPolicy::Fail => Either::Left(id), + AffinityPolicy::Allow => Either::Right(id), + } + }); + let (required, mut preferred): (HashSet<_>, HashSet<_>) = + affinity_sleds.into_iter().partition_map(|(policy, id)| { + let id = SledUuid::from_untyped_uuid(id); + match policy { + AffinityPolicy::Fail => Either::Left(id), + AffinityPolicy::Allow => Either::Right(id), + } + }); + + if !banned.is_empty() { + info!( + log, + "anti-affinity policy prohibits placement on {} sleds", banned.len(); + "banned" => ?banned, + ); + } + if !required.is_empty() { + info!( + log, + "affinity policy requires placement on {} sleds", required.len(); + "required" => ?required, + ); + } + + if required.len() > 1 { + return Err(SledReservationError::TooManyAffinityConstraints); + } + if let Some(required_id) = required.iter().next() { + // If we have a "required" sled, it must be chosen. + if banned.contains(&required_id) { + return Err( + SledReservationError::ConflictingAntiAndAffinityConstraints, + ); + } + if !targets.contains(&required_id) { + return Err(SledReservationError::RequiredAffinitySledNotValid); + } + return Ok(*required_id); + } + + // We have no "required" sleds, but might have preferences. + targets = targets.difference(&banned).cloned().collect(); + + // Only consider "preferred" sleds that are viable targets + preferred = targets.intersection(&preferred).cloned().collect(); + // Only consider "unpreferred" sleds that are viable targets + unpreferred = targets.intersection(&unpreferred).cloned().collect(); + + // If a target is both preferred and unpreferred, it is removed + // from both sets. + let both = preferred.intersection(&unpreferred).cloned().collect(); + preferred = preferred.difference(&both).cloned().collect(); + unpreferred = unpreferred.difference(&both).cloned().collect(); + + if let Some(target) = preferred.iter().next() { + return Ok(*target); + } + targets = targets.difference(&unpreferred).cloned().collect(); + if let Some(target) = targets.iter().next() { + return Ok(*target); + } + if let Some(target) = unpreferred.iter().next() { + return Ok(*target); + } + return Err(SledReservationError::NotFound); +} + impl DataStore { /// Stores a new sled in the database. /// @@ -231,18 +440,7 @@ impl DataStore { SledReservationTransactionError::Diesel(e) => { public_error_from_diesel(e, ErrorHandler::Server) } - SledReservationTransactionError::Reservation(e) => match e { - SledReservationError::NotFound - | SledReservationError::TooManyAffinityConstraints - | SledReservationError::ConflictingAntiAndAffinityConstraints - | SledReservationError::RequiredAffinitySledNotValid => { - return external::Error::insufficient_capacity( - "No sleds can fit the requested instance", - "No sled targets found that had enough \ - capacity to fit the requested instance.", - ); - } - }, + SledReservationTransactionError::Reservation(e) => e.into(), }) } @@ -367,177 +565,26 @@ impl DataStore { instance_id, ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; - // We use the following logic to calculate a desirable sled, - // given a possible set of "targets", and the information - // from affinity groups. - // - // # Rules vs Preferences - // - // Due to the flavors "affinity policy", it's possible to bucket affinity - // choices into two categories: "rules" and "preferences". "rules" are affinity - // dispositions for or against sled placement that must be followed, and - // "preferences" are affinity dispositions that should be followed for sled - // selection, in order of "most preferential" to "least preferential". - // - // As example of a "rule" is "an anti-affinity group exists, containing a - // target sled, with affinity_policy = 'fail'". - // - // An example of a "preference" is "an anti-affinity group exists, containing a - // target sled, but the policy is 'allow'. We don't want to use it as a target, - // but we will if there are no other choices." - // - // We apply rules before preferences to ensure they are always respected. - // Furthermore, the evaluation of preferences is a target-seeking operation, - // which identifies the distinct sets of targets, and searches them in - // decreasing preference order. - // - // # Logic - // - // ## Background: Notation - // - // We use the following symbols for sets below: - // - ∩: Intersection of two sets (A ∩ B is "everything that exists in A and - // also exists in B"). - // - \: difference of two sets (A \ B is "everything that exists in A that does - // not exist in B). - // - // We also use the following notation for brevity: - // - AA,P=Fail: All sleds sharing an anti-affinity instance within a group with - // policy = 'fail'. - // - AA,P=Allow: Same as above, but with policy = 'allow'. - // - A,P=Fail: All sleds sharing an affinity instance within a group with - // policy = 'fail'. - // - A,P=Allow: Same as above, but with policy = 'allow'. - // - // ## Affinity: Apply Rules - // - // - Targets := All viable sleds for instance placement - // - Banned := AA,P=Fail - // - Required := A,P=Fail - // - if Required.len() > 1: Fail (too many constraints). - // - if Required.len() == 1... - // - ... if the entry exists in the "Banned" set: Fail - // (contradicting constraints 'Banned' + 'Required') - // - ... if the entry does not exist in "Targets": Fail - // ('Required' constraint not satisfiable) - // - ... if the entry does not exist in "Banned": Use it. - // - // If we have not yet picked a target, we can filter the - // set of targets to ignore "banned" sleds, and then apply - // preferences. - // - // - Targets := Targets \ Banned - // - // ## Affinity: Apply Preferences - // - // - Preferred := Targets ∩ A,P=Allow - // - Unpreferred := Targets ∩ AA,P=Allow - // - Neither := Preferred ∩ Unpreferred - // - Preferred := Preferred \ Neither - // - Unpreferred := Unpreferred \ Neither - // - If Preferred isn't empty, pick a target from it. - // - Targets := Targets \ Unpreferred - // - If Targets isn't empty, pick a target from it. - // - If Unpreferred isn't empty, pick a target from it. - // - Fail, no targets are available. - - let mut targets: HashSet<_> = sled_targets.into_iter().collect(); - - let banned = anti_affinity_sleds.iter().filter_map(|(policy, id)| { - if *policy == AffinityPolicy::Fail { - Some(*id) - } else { - None - } - }).collect::>(); - let required = affinity_sleds.iter().filter_map(|(policy, id)| { - if *policy == AffinityPolicy::Fail { - Some(*id) - } else { - None - } - }).collect::>(); - - if !banned.is_empty() { - info!( - opctx.log, - "affinity policy prohibits placement on {} sleds", banned.len(); - "banned" => ?banned, - ); - } - if !required.is_empty() { - info!( - opctx.log, - "affinity policy requires placement on {} sleds", required.len(); - "required" => ?required, - ); - } - - let sled_target = if required.len() > 1 { - return Err(err.bail(SledReservationError::TooManyAffinityConstraints)); - } else if let Some(required_id) = required.iter().next() { - // If we have a "required" sled, it must be chosen. - - if banned.contains(&required_id) { - return Err(err.bail(SledReservationError::ConflictingAntiAndAffinityConstraints)); - } - if !targets.contains(&required_id) { - return Err(err.bail(SledReservationError::RequiredAffinitySledNotValid)); - } - *required_id - } else { - // We have no "required" sleds, but might have preferences. + let targets: HashSet = sled_targets + .into_iter() + .map(|id| SledUuid::from_untyped_uuid(id)) + .collect(); - targets = targets.difference(&banned).cloned().collect(); - - let mut preferred = affinity_sleds.iter().filter_map(|(policy, id)| { - if *policy == AffinityPolicy::Allow { - Some(*id) - } else { - None - } - }).collect::>(); - let mut unpreferred = anti_affinity_sleds.iter().filter_map(|(policy, id)| { - if *policy == AffinityPolicy::Allow { - Some(*id) - } else { - None - } - }).collect::>(); - - // Only consider "preferred" sleds that are viable targets - preferred = targets.intersection(&preferred).cloned().collect(); - // Only consider "unpreferred" sleds that are viable targets - unpreferred = targets.intersection(&unpreferred).cloned().collect(); - - // If a target is both preferred and unpreferred, it is removed - // from both sets. - let both = preferred.intersection(&unpreferred).cloned().collect(); - preferred = preferred.difference(&both).cloned().collect(); - unpreferred = unpreferred.difference(&both).cloned().collect(); - - if let Some(target) = preferred.iter().next() { - *target - } else { - targets = targets.difference(&unpreferred).cloned().collect(); - if let Some(target) = targets.iter().next() { - *target - } else { - if let Some(target) = unpreferred.iter().next() { - *target - } else { - return Err(err.bail(SledReservationError::NotFound)); - } - } - } - }; + let sled_target = pick_sled_reservation_target( + &opctx.log, + targets, + anti_affinity_sleds, + affinity_sleds, + ).map_err(|e| { + err.bail(e) + })?; // Create a SledResource record, associate it with the target // sled. let resource = SledResource::new_for_vmm( propolis_id, instance_id, - SledUuid::from_untyped_uuid(sled_target), + sled_target, resources, ); @@ -1082,6 +1129,8 @@ pub(in crate::db::datastore) mod test { use nexus_types::identity::Resource; use omicron_common::api::external; use omicron_test_utils::dev; + use omicron_uuid_kinds::AffinityGroupUuid; + use omicron_uuid_kinds::AntiAffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; @@ -1451,20 +1500,16 @@ pub(in crate::db::datastore) mod test { // creating affinity groups, but makes testing easier. async fn add_instance_to_anti_affinity_group( datastore: &DataStore, - group_id: Uuid, - instance_id: Uuid, + group_id: AntiAffinityGroupUuid, + instance_id: InstanceUuid, ) { use db::model::AntiAffinityGroupInstanceMembership; use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; - use omicron_uuid_kinds::AntiAffinityGroupUuid; diesel::insert_into( membership_dsl::anti_affinity_group_instance_membership, ) - .values(AntiAffinityGroupInstanceMembership::new( - AntiAffinityGroupUuid::from_untyped_uuid(group_id), - InstanceUuid::from_untyped_uuid(instance_id), - )) + .values(AntiAffinityGroupInstanceMembership::new(group_id, instance_id)) .on_conflict((membership_dsl::group_id, membership_dsl::instance_id)) .do_nothing() .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) @@ -1503,18 +1548,14 @@ pub(in crate::db::datastore) mod test { // creating affinity groups, but makes testing easier. async fn add_instance_to_affinity_group( datastore: &DataStore, - group_id: Uuid, - instance_id: Uuid, + group_id: AffinityGroupUuid, + instance_id: InstanceUuid, ) { use db::model::AffinityGroupInstanceMembership; use db::schema::affinity_group_instance_membership::dsl as membership_dsl; - use omicron_uuid_kinds::AffinityGroupUuid; diesel::insert_into(membership_dsl::affinity_group_instance_membership) - .values(AffinityGroupInstanceMembership::new( - AffinityGroupUuid::from_untyped_uuid(group_id), - InstanceUuid::from_untyped_uuid(instance_id), - )) + .values(AffinityGroupInstanceMembership::new(group_id, instance_id)) .on_conflict(( membership_dsl::group_id, membership_dsl::instance_id, @@ -1667,16 +1708,16 @@ pub(in crate::db::datastore) mod test { Affinity::Positive => { add_instance_to_affinity_group( &datastore, - *group_id, - self.id.into_untyped_uuid(), + AffinityGroupUuid::from_untyped_uuid(*group_id), + self.id, ) .await } Affinity::Negative => { add_instance_to_anti_affinity_group( &datastore, - *group_id, - self.id.into_untyped_uuid(), + AntiAffinityGroupUuid::from_untyped_uuid(*group_id), + self.id, ) .await } diff --git a/nexus/db-queries/src/db/queries/affinity.rs b/nexus/db-queries/src/db/queries/affinity.rs index 6da857669da..24806fe85d8 100644 --- a/nexus/db-queries/src/db/queries/affinity.rs +++ b/nexus/db-queries/src/db/queries/affinity.rs @@ -10,9 +10,11 @@ use diesel::sql_types; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; -/// For an instance, look up all anti-affinity groups, and return -/// a list of sleds with other instances in that anti-affinity group -/// already reserved. +/// For an instance, look up all anti-affinity groups it belongs to. +/// For all those groups, find all instances with reservations, and look +/// up their sleds. +/// Return all the sleds on which those instances were found, along with +/// policy information about the corresponding group. pub fn lookup_anti_affinity_sleds_query( instance_id: InstanceUuid, ) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { @@ -49,8 +51,11 @@ pub fn lookup_anti_affinity_sleds_query( .query() } -/// For an instance, look up all affinity groups, and return a list of sleds -/// with other instances in that affinity group already reserved. +/// For an instance, look up all affinity groups it belongs to. +/// For all those groups, find all instances with reservations, and look +/// up their sleds. +/// Return all the sleds on which those instances were found, along with +/// policy information about the corresponding group. pub fn lookup_affinity_sleds_query( instance_id: InstanceUuid, ) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { From fdccd6bd041f501d4e03d5850e161b726b284881 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 11:31:17 -0800 Subject: [PATCH 21/84] review feedback (grammatical) --- nexus/db-queries/src/db/datastore/sled.rs | 31 +++++++++++++---------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index bb61d8eebac..217ab71718a 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -61,25 +61,28 @@ enum SledReservationError { "Could not find any valid sled on which this instance can be placed" )] NotFound, - #[error("This instance belongs to an affinity group that requires it be placed \ - on more than one sled. Instances can only placed on a single sled, so \ - this is impossible to satisfy - Consider stopping other instances in \ - the affinity group.")] + #[error( + "This instance belongs to an affinity group that requires it be placed \ + on more than one sled. Instances can only placed on a single sled, so \ + this is impossible to satisfy. Consider stopping other instances in \ + the affinity group." + )] TooManyAffinityConstraints, #[error( "This instance belongs to an affinity group that requires it to be \ - placed on a sled, but also belongs to an anti-affinity group that \ - prevents it from being placed on that sled. These constraints are \ - contradictory: Consider stopping instances in those \ - affinity/anti-affinity groups, or changing group membership." + placed on a sled, but also belongs to an anti-affinity group that \ + prevents it from being placed on that sled. These constraints are \ + contradictory. Consider stopping instances in those \ + affinity/anti-affinity groups, or changing group membership." )] ConflictingAntiAndAffinityConstraints, - #[error("This instance must be placed on a specific sled due to affinity \ - rules, but that placement is invalid for some other reason (e.g., \ - the instance would not fit, or the sled sharing an instance in the \ - affinity group is being expunged). Consider stopping other \ - instances in the affinity group, or changing affinity group \ - membership.")] + #[error( + "This instance must be placed on a specific sled to co-locate it \ + with another instance in its affinity group, but that sled cannot \ + current accept this instance. Consider stopping other instances \ + in this instance's affinity groups, or changing its affinity \ + group membership." + )] RequiredAffinitySledNotValid, } From 6e443925fda94f956978b59888a8600021f09db9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 11:51:55 -0800 Subject: [PATCH 22/84] anti-affinity group description --- nexus/test-utils/src/resource_helpers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 3ef1858cfa2..011c4a72773 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -616,7 +616,7 @@ pub async fn create_anti_affinity_group( ¶ms::AntiAffinityGroupCreate { identity: IdentityMetadataCreateParams { name: group_name.parse().unwrap(), - description: String::from("affinity group description"), + description: String::from("anti-affinity group description"), }, policy: AffinityPolicy::Fail, failure_domain: FailureDomain::Sled, From d5b951fd77cf83a0f3f9763a16189860a1a4595d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 12:04:44 -0800 Subject: [PATCH 23/84] Update comment --- nexus/tests/integration_tests/affinity.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/tests/integration_tests/affinity.rs b/nexus/tests/integration_tests/affinity.rs index 65ab96b0a90..1549db51cba 100644 --- a/nexus/tests/integration_tests/affinity.rs +++ b/nexus/tests/integration_tests/affinity.rs @@ -623,7 +623,7 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { .map(|instance| instance.identity.id) .collect::>(); - // We expect that each sled will have a since instance, as none of the + // We expect that each sled will have a since instance, as all of the // instances will want to be anti-located from each other. for sled in &sleds { let observed_instances = api From f57a96acfaca915dbc50507fd950bc3196c3b24d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 12:41:38 -0800 Subject: [PATCH 24/84] Fix schema migration --- schema/crdb/sled-resource-for-vmm/up05.sql | 2 +- schema/crdb/sled-resource-for-vmm/up06.sql | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 schema/crdb/sled-resource-for-vmm/up06.sql diff --git a/schema/crdb/sled-resource-for-vmm/up05.sql b/schema/crdb/sled-resource-for-vmm/up05.sql index cfbb7ce8fba..6ee16589037 100644 --- a/schema/crdb/sled-resource-for-vmm/up05.sql +++ b/schema/crdb/sled-resource-for-vmm/up05.sql @@ -1 +1 @@ -ALTER INDEX IF EXISTS lookup_resource_by_vmm RENAME TO lookup_vmm_resource_by_instance; +ALTER INDEX IF EXISTS lookup_resource_by_instance RENAME TO lookup_vmm_resource_by_instance; diff --git a/schema/crdb/sled-resource-for-vmm/up06.sql b/schema/crdb/sled-resource-for-vmm/up06.sql new file mode 100644 index 00000000000..57bec13baa1 --- /dev/null +++ b/schema/crdb/sled-resource-for-vmm/up06.sql @@ -0,0 +1,2 @@ +ALTER INDEX IF EXISTS omicron.public.sled_resource_vmm @ sled_resource_pkey + RENAME TO sled_resource_vmm_pkey; From fb8976d80bc00dcb9d2086ecec933b8fe87d5f07 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 15:55:45 -0800 Subject: [PATCH 25/84] Redo regression test for OID poisoning, and move it --- nexus/tests/integration_tests/schema.rs | 279 ++++++++++++++++++++---- 1 file changed, 233 insertions(+), 46 deletions(-) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 0e0d4832718..618c7ad1794 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -22,6 +22,7 @@ use similar_asserts; use slog::Logger; use std::collections::BTreeMap; use std::net::IpAddr; +use std::sync::Mutex; use tokio::time::timeout; use tokio::time::Duration; use uuid::Uuid; @@ -976,14 +977,97 @@ async fn dbinit_equals_sum_of_all_up() { logctx.cleanup_successful(); } -type BeforeFn = for<'a> fn(client: &'a Client) -> BoxFuture<'a, ()>; -type AfterFn = for<'a> fn(client: &'a Client) -> BoxFuture<'a, ()>; +type Conn = qorb::claim::Handle< + async_bb8_diesel::Connection, +>; + +struct PoolAndConnection { + pool: nexus_db_queries::db::Pool, + conn: Conn, +} + +impl PoolAndConnection { + async fn cleanup(self) { + drop(self.conn); + self.pool.terminate().await; + } +} + +struct MigrationContext<'a> { + log: &'a Logger, + + // Postgres connection to database + client: Client, + + // Reference to the database itself + crdb: &'a CockroachInstance, + + // An optionally-populated "pool and connection" from before + // a schema version upgrade. + // + // This can be used to validate properties of a database pool + // before and after a particular schema migration. + pool_and_conn: Mutex>, +} + +impl<'a> MigrationContext<'a> { + // Populates a pool and connection. + // + // Typically called as a part of a "before" function, to set up a connection + // before a schema migration. + async fn populate_pool_and_connection(&self, version: SemverVersion) { + let pool = nexus_db_queries::db::Pool::new_single_host( + self.log, + &nexus_db_queries::db::Config { + url: self.crdb.pg_config().clone(), + }, + ); + let conn = pool.claim().await.expect("failed to get pooled connection"); + + let mut map = self.pool_and_conn.lock().unwrap(); + map.insert(version, PoolAndConnection { pool, conn }); + } + + // Takes a pool and connection if they've been populated. + fn take_pool_and_connection( + &self, + version: &SemverVersion, + ) -> PoolAndConnection { + let mut map = self.pool_and_conn.lock().unwrap(); + map.remove(version).unwrap() + } +} + +type BeforeFn = for<'a> fn(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()>; +type AfterFn = for<'a> fn(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()>; +type AtCurrentFn = + for<'a> fn(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()>; // Describes the operations which we might take before and after // migrations to check that they worked. +#[derive(Default)] struct DataMigrationFns { before: Option, - after: AfterFn, + after: Option, + at_current: Option, +} + +impl DataMigrationFns { + fn new() -> Self { + Self::default() + } + fn before(mut self, before: BeforeFn) -> Self { + self.before = Some(before); + self + } + fn after(mut self, after: AfterFn) -> Self { + self.after = Some(after); + self + } + fn at_current(mut self, at_current: AtCurrentFn) -> Self { + self.at_current = Some(at_current); + self + } } // "51F0" -> "Silo" @@ -1028,10 +1112,10 @@ const VOLUME2: Uuid = Uuid::from_u128(0x2222566f_5c3d_4647_83b0_8f3515da7be1); const VOLUME3: Uuid = Uuid::from_u128(0x3333566f_5c3d_4647_83b0_8f3515da7be1); const VOLUME4: Uuid = Uuid::from_u128(0x4444566f_5c3d_4647_83b0_8f3515da7be1); -fn before_23_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn before_23_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async move { // Create two silos - client.batch_execute(&format!("INSERT INTO silo + ctx.client.batch_execute(&format!("INSERT INTO silo (id, name, description, time_created, time_modified, time_deleted, discoverable, authentication_mode, user_provision_type, mapped_fleet_roles, rcgen) VALUES ('{SILO1}', 'silo1', '', now(), now(), NULL, false, 'local', 'jit', '{{}}', 1), ('{SILO2}', 'silo2', '', now(), now(), NULL, false, 'local', 'jit', '{{}}', 1); @@ -1039,7 +1123,7 @@ fn before_23_0_0(client: &Client) -> BoxFuture<'_, ()> { // Create an IP pool for each silo, and a third "fleet pool" which has // no corresponding silo. - client.batch_execute(&format!("INSERT INTO ip_pool + ctx.client.batch_execute(&format!("INSERT INTO ip_pool (id, name, description, time_created, time_modified, time_deleted, rcgen, silo_id, is_default) VALUES ('{POOL0}', 'pool2', '', now(), now(), now(), 1, '{SILO2}', true), ('{POOL1}', 'pool1', '', now(), now(), NULL, 1, '{SILO1}', true), @@ -1050,11 +1134,12 @@ fn before_23_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn after_23_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_23_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { // Confirm that the ip_pool_resource objects have been created // by the migration. - let rows = client + let rows = ctx + .client .query("SELECT * FROM ip_pool_resource ORDER BY ip_pool_id", &[]) .await .expect("Failed to query ip pool resource"); @@ -1103,12 +1188,12 @@ fn after_23_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn before_24_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { // IP addresses were pulled off dogfood sled 16 Box::pin(async move { // Create two sleds. (SLED2 is marked non_provisionable for // after_37_0_1.) - client + ctx.client .batch_execute(&format!( "INSERT INTO sled (id, time_created, time_modified, time_deleted, rcgen, rack_id, @@ -1129,10 +1214,11 @@ fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn after_24_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_24_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { // Confirm that the IP Addresses have the last 2 bytes changed to `0xFFFF` - let rows = client + let rows = ctx + .client .query("SELECT last_used_address FROM sled ORDER BY id", &[]) .await .expect("Failed to sled last_used_address"); @@ -1155,10 +1241,11 @@ fn after_24_0_0(client: &Client) -> BoxFuture<'_, ()> { } // This reuses the sleds created in before_24_0_0. -fn after_37_0_1(client: &Client) -> BoxFuture<'_, ()> { +fn after_37_0_1<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { // Confirm that the IP Addresses have the last 2 bytes changed to `0xFFFF` - let rows = client + let rows = ctx + .client .query("SELECT sled_policy, sled_state FROM sled ORDER BY id", &[]) .await .expect("Failed to select sled policy and state"); @@ -1193,9 +1280,9 @@ fn after_37_0_1(client: &Client) -> BoxFuture<'_, ()> { }) } -fn before_70_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn before_70_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async move { - client + ctx.client .batch_execute(&format!( " INSERT INTO instance (id, name, description, time_created, @@ -1218,7 +1305,7 @@ fn before_70_0_0(client: &Client) -> BoxFuture<'_, ()> { .await .expect("failed to create instances"); - client + ctx.client .batch_execute(&format!( " INSERT INTO vmm (id, time_created, time_deleted, instance_id, state, @@ -1234,9 +1321,10 @@ fn before_70_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn after_70_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_70_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { - let rows = client + let rows = ctx + .client .query("SELECT state FROM instance ORDER BY id", &[]) .await .expect("failed to load instance states"); @@ -1264,7 +1352,8 @@ fn after_70_0_0(client: &Client) -> BoxFuture<'_, ()> { )] ); - let rows = client + let rows = ctx + .client .query("SELECT state FROM vmm ORDER BY id", &[]) .await .expect("failed to load VMM states"); @@ -1280,11 +1369,12 @@ fn after_70_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn before_95_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn before_95_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { // This reuses the instance records created in `before_70_0_0` const COLUMN: &'static str = "boot_on_fault"; Box::pin(async { - let rows = client + let rows = ctx + .client .query("SELECT boot_on_fault FROM instance ORDER BY id", &[]) .await .expect("failed to load instance boot_on_fault settings"); @@ -1308,10 +1398,11 @@ fn before_95_0_0(client: &Client) -> BoxFuture<'_, ()> { const COLUMN_AUTO_RESTART: &'static str = "auto_restart_policy"; const ENUM_AUTO_RESTART: &'static str = "instance_auto_restart"; -fn after_95_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_95_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { // This reuses the instance records created in `before_70_0_0` Box::pin(async { - let rows = client + let rows = ctx + .client .query( &format!( "SELECT {COLUMN_AUTO_RESTART} FROM instance ORDER BY id" @@ -1340,11 +1431,11 @@ fn after_95_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn before_101_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn before_101_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { // Make a new instance with an explicit 'sled_failures_only' v1 auto-restart // policy - client + ctx.client .batch_execute(&format!( "INSERT INTO instance ( id, name, description, time_created, time_modified, @@ -1362,7 +1453,7 @@ fn before_101_0_0(client: &Client) -> BoxFuture<'_, ()> { .await .expect("failed to create instance"); // Change one of the NULLs to an explicit 'never'. - client + ctx.client .batch_execute(&format!( "UPDATE instance SET {COLUMN_AUTO_RESTART} = 'never' @@ -1370,13 +1461,23 @@ fn before_101_0_0(client: &Client) -> BoxFuture<'_, ()> { )) .await .expect("failed to update instance"); + + // Used as a regression test against + // https://github.com/oxidecomputer/omicron/issues/5561 + // + // See 'at_current_101_0_0' - we create a connection here, because connections + // may be populating an OID cache. We use the connection after the + // schema migration to access the "instance_auto_restart" type. + let semver = SemverVersion(semver::Version::parse("101.0.0").unwrap()); + ctx.populate_pool_and_connection(semver).await; }) } -fn after_101_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_101_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { const BEST_EFFORT: &'static str = "best_effort"; Box::pin(async { - let rows = client + let rows = ctx + .client .query( &format!( "SELECT {COLUMN_AUTO_RESTART} FROM instance ORDER BY id" @@ -1421,12 +1522,82 @@ fn after_101_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn before_107_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn at_current_101_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { + Box::pin(async { + // Used as a regression test against + // https://github.com/oxidecomputer/omicron/issues/5561 + // + // See 'before_101_0_0' - we created a connection to validate that we can + // access the "instance_auto_restart" type without encountering + // OID cache poisoning. + // + // We care about the following: + // + // 1. This type was dropped and re-created in some schema migration, + // 2. The type still exists in the latest schema + // + // This can happen on any schema migration, but it happens to exist + // here. To find other schema migrations where this might be possible, + // try the following search from within the "omicron/schema/crdb" + // directory: + // + // ```bash + // rg 'DROP TYPE IF EXISTS (.*);' --no-filename -o --replace '$1' + // | sort -u + // | xargs -I {} rg 'CREATE TYPE .*{}' dbinit.sql + // ``` + // + // This finds all user-defined types which have dropped at some point, + // but which still appear in the latest schema. + let semver = SemverVersion(semver::Version::parse("101.0.0").unwrap()); + let pool_and_conn = ctx.take_pool_and_connection(&semver); + + { + use async_bb8_diesel::AsyncRunQueryDsl; + use nexus_db_model::schema::instance::dsl; + use nexus_db_model::Instance; + use nexus_types::external_api::params; + use omicron_common::api::external::IdentityMetadataCreateParams; + use omicron_uuid_kinds::InstanceUuid; + + diesel::insert_into(dsl::instance) + .values(Instance::new( + InstanceUuid::new_v4(), + Uuid::new_v4(), + ¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: "hello".parse().unwrap(), + description: "hello".to_string(), + }, + ncpus: 2.try_into().unwrap(), + memory: 1024_u64.try_into().unwrap(), + hostname: "inst".parse().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: + params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![], + boot_disk: None, + disks: Vec::new(), + start: false, + auto_restart_policy: Default::default(), + }, + )) + .execute_async(&*pool_and_conn.conn) + .await + .expect("failed to insert - did we poison the OID cache?"); + } + + pool_and_conn.cleanup().await; + }) +} + +fn before_107_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { // An instance with no attached disks (4) gets a NULL boot disk. // An instance with one attached disk (5) gets that disk as a boot disk. // An instance with two attached disks (6) gets a NULL boot disk. - client + ctx.client .batch_execute(&format!( " INSERT INTO disk ( @@ -1463,9 +1634,10 @@ fn before_107_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn after_107_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_107_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { - let rows = client + let rows = ctx + .client .query("SELECT id, boot_disk_id FROM instance ORDER BY id;", &[]) .await .expect("failed to load instance boot disks"); @@ -1512,6 +1684,7 @@ fn after_107_0_0(client: &Client) -> BoxFuture<'_, ()> { ); }) } + // Lazily initializes all migration checks. The combination of Rust function // pointers and async makes defining a static table fairly painful, so we're // using lazy initialization instead. @@ -1523,33 +1696,34 @@ fn get_migration_checks() -> BTreeMap { map.insert( SemverVersion(semver::Version::parse("23.0.0").unwrap()), - DataMigrationFns { before: Some(before_23_0_0), after: after_23_0_0 }, + DataMigrationFns::new().before(before_23_0_0).after(after_23_0_0), ); map.insert( SemverVersion(semver::Version::parse("24.0.0").unwrap()), - DataMigrationFns { before: Some(before_24_0_0), after: after_24_0_0 }, + DataMigrationFns::new().before(before_24_0_0).after(after_24_0_0), ); map.insert( SemverVersion(semver::Version::parse("37.0.1").unwrap()), - DataMigrationFns { before: None, after: after_37_0_1 }, + DataMigrationFns::new().after(after_37_0_1), ); map.insert( SemverVersion(semver::Version::parse("70.0.0").unwrap()), - DataMigrationFns { before: Some(before_70_0_0), after: after_70_0_0 }, + DataMigrationFns::new().before(before_70_0_0).after(after_70_0_0), ); map.insert( SemverVersion(semver::Version::parse("95.0.0").unwrap()), - DataMigrationFns { before: Some(before_95_0_0), after: after_95_0_0 }, + DataMigrationFns::new().before(before_95_0_0).after(after_95_0_0), ); - map.insert( SemverVersion(semver::Version::parse("101.0.0").unwrap()), - DataMigrationFns { before: Some(before_101_0_0), after: after_101_0_0 }, + DataMigrationFns::new() + .before(before_101_0_0) + .after(after_101_0_0) + .at_current(at_current_101_0_0), ); - map.insert( SemverVersion(semver::Version::parse("107.0.0").unwrap()), - DataMigrationFns { before: Some(before_107_0_0), after: after_107_0_0 }, + DataMigrationFns::new().before(before_107_0_0).after(after_107_0_0), ); map @@ -1583,7 +1757,12 @@ async fn validate_data_migration() { let db = TestDatabase::new_populate_nothing(&logctx.log).await; let crdb = db.crdb(); - let client = crdb.connect().await.expect("Failed to access CRDB client"); + let ctx = MigrationContext { + log, + client: crdb.connect().await.expect("Failed to access CRDB client"), + crdb, + pool_and_conn: Mutex::new(BTreeMap::new()), + }; let all_versions = read_all_schema_versions(); let all_checks = get_migration_checks(); @@ -1593,7 +1772,7 @@ async fn validate_data_migration() { // If this check has preconditions (or setup), run them. let checks = all_checks.get(version.semver()); if let Some(before) = checks.and_then(|check| check.before) { - before(&client).await; + before(&ctx).await; } apply_update(log, &crdb, version, 1).await; @@ -1603,8 +1782,8 @@ async fn validate_data_migration() { ); // If this check has postconditions (or cleanup), run them. - if let Some(after) = checks.map(|check| check.after) { - after(&client).await; + if let Some(after) = checks.and_then(|check| check.after) { + after(&ctx).await; } } assert_eq!( @@ -1612,6 +1791,14 @@ async fn validate_data_migration() { query_crdb_schema_version(&crdb).await ); + // If any version changes want to query the system post-upgrade, they can. + for version in all_versions.iter_versions() { + let checks = all_checks.get(version.semver()); + if let Some(at_current) = checks.and_then(|check| check.at_current) { + at_current(&ctx).await; + } + } + db.terminate().await; logctx.cleanup_successful(); } From 17dc409a68b1e14961b088d286661d7c6337c919 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 16:01:02 -0800 Subject: [PATCH 26/84] clippppyyyyyy --- nexus/tests/integration_tests/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 618c7ad1794..6a5ce8ed0a3 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -1010,7 +1010,7 @@ struct MigrationContext<'a> { pool_and_conn: Mutex>, } -impl<'a> MigrationContext<'a> { +impl MigrationContext<'_> { // Populates a pool and connection. // // Typically called as a part of a "before" function, to set up a connection From 97b9b2e7599d65347a4b31ea24f457f63cfd9f49 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Feb 2025 16:16:22 -0800 Subject: [PATCH 27/84] Eliza's feedback --- nexus/db-queries/src/db/datastore/sled.rs | 24 +++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 217ab71718a..9621515e17d 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -267,21 +267,25 @@ fn pick_sled_reservation_target( // Only consider "unpreferred" sleds that are viable targets unpreferred = targets.intersection(&unpreferred).cloned().collect(); - // If a target is both preferred and unpreferred, it is removed - // from both sets. + // If a target is both preferred and unpreferred, it is not considered + // a part of either set. let both = preferred.intersection(&unpreferred).cloned().collect(); - preferred = preferred.difference(&both).cloned().collect(); - unpreferred = unpreferred.difference(&both).cloned().collect(); - if let Some(target) = preferred.iter().next() { - return Ok(*target); + // Grab a preferred target (which isn't also unpreferred) if one exists. + if let Some(target) = preferred.difference(&both).cloned().next() { + return Ok(target); } + unpreferred = unpreferred.difference(&both).cloned().collect(); targets = targets.difference(&unpreferred).cloned().collect(); - if let Some(target) = targets.iter().next() { - return Ok(*target); + + // Grab a target which not in the unpreferred set, if one exists. + if let Some(target) = targets.iter().cloned().next() { + return Ok(target); } - if let Some(target) = unpreferred.iter().next() { - return Ok(*target); + + // Grab a target from the unpreferred set, if one exists. + if let Some(target) = unpreferred.iter().cloned().next() { + return Ok(target); } return Err(SledReservationError::NotFound); } From 79b425200b6ace52bd5d600b613bf276fbcf2779 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Feb 2025 14:09:50 -0800 Subject: [PATCH 28/84] [nexus-db-queries] Benchmark for VMM reservation --- Cargo.lock | 1 + nexus/db-queries/Cargo.toml | 7 +- .../benches/sled_reservation_benchmark.rs | 346 ++++++++++++++++++ 3 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 nexus/db-queries/benches/sled_reservation_benchmark.rs diff --git a/Cargo.lock b/Cargo.lock index 602ae6a21fa..a1e4dcc1f9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5858,6 +5858,7 @@ dependencies = [ "chrono", "clickhouse-admin-types", "const_format", + "criterion", "db-macros", "diesel", "diesel-dtrace", diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index f51629f63cd..a63cfcc223a 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -76,6 +76,7 @@ testing = ["omicron-test-utils"] [dev-dependencies] assert_matches.workspace = true camino-tempfile.workspace = true +criterion.workspace = true expectorate.workspace = true hyper-rustls.workspace = true gateway-client.workspace = true @@ -86,7 +87,7 @@ nexus-inventory.workspace = true nexus-reconfigurator-planning.workspace = true nexus-test-utils.workspace = true omicron-sled-agent.workspace = true -omicron-test-utils.workspace = true +omicron-test-utils = { workspace = true, features = ["seed-gen"] } openapiv3.workspace = true oso.workspace = true pem.workspace = true @@ -98,3 +99,7 @@ regex.workspace = true rustls.workspace = true subprocess.workspace = true term.workspace = true + +[[bench]] +name = "sled_reservation_benchmark" +harness = false diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs new file mode 100644 index 00000000000..4d9c8bae717 --- /dev/null +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -0,0 +1,346 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Benchmarks creating sled reservations + +use criterion::black_box; +use criterion::{criterion_group, criterion_main, Criterion}; +use nexus_db_model::ByteCount; +use nexus_db_model::Generation; +use nexus_db_model::Project; +use nexus_db_model::Resources; +use nexus_db_model::Sled; +use nexus_db_model::SledBaseboard; +use nexus_db_model::SledReservationConstraintBuilder; +use nexus_db_model::SledSystemHardware; +use nexus_db_model::SledUpdate; +use nexus_db_queries::authz; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::pub_test_utils::TestDatabase; +use nexus_db_queries::db::DataStore; +use nexus_types::external_api::params; +use omicron_common::api::external; +use omicron_test_utils::dev; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use slog::Logger; +use std::net::Ipv6Addr; +use std::net::SocketAddrV6; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; +use uuid::Uuid; + +///////////////////////////////////////////////////////////////// +// +// TEST HELPERS +// +// These are largely ripped out of "nexus/db-queries/src/db/datastore". +// +// Benchmarks are compiled as external binaries from library crates, so we +// can only access `pub` code. +// +// It may be worth refactoring some of these functions to a test utility +// crate to avoid the de-duplication. + +async fn create_project( + opctx: &OpContext, + datastore: &DataStore, +) -> (authz::Project, Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + + // Create a project + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: external::IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: "desc".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() +} + +fn rack_id() -> Uuid { + Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap() +} + +// Creates a "fake" Sled Baseboard. +fn sled_baseboard_for_test() -> SledBaseboard { + SledBaseboard { + serial_number: Uuid::new_v4().to_string(), + part_number: String::from("test-part"), + revision: 1, + } +} + +// Creates "fake" sled hardware accounting +fn sled_system_hardware_for_test() -> SledSystemHardware { + SledSystemHardware { + is_scrimlet: false, + usable_hardware_threads: 4, + usable_physical_ram: ByteCount::try_from(1 << 40).unwrap(), + reservoir_size: ByteCount::try_from(1 << 39).unwrap(), + } +} + +fn test_new_sled_update() -> SledUpdate { + let sled_id = Uuid::new_v4(); + let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); + let repo_depot_port = 0; + SledUpdate::new( + sled_id, + addr, + repo_depot_port, + sled_baseboard_for_test(), + sled_system_hardware_for_test(), + rack_id(), + Generation::new(), + ) +} + +async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { + let mut sleds = vec![]; + for _ in 0..count { + let (sled, _) = + datastore.sled_upsert(test_new_sled_update()).await.unwrap(); + sleds.push(sled); + } + sleds +} + +fn small_resource_request() -> Resources { + Resources::new( + 1, + // Just require the bare non-zero amount of RAM. + ByteCount::try_from(1024).unwrap(), + ByteCount::try_from(1024).unwrap(), + ) +} + +async fn create_reservation(opctx: &OpContext, db: &DataStore) -> PropolisUuid { + let instance_id = InstanceUuid::new_v4(); + let vmm_id = PropolisUuid::new_v4(); + db.sled_reservation_create( + &opctx, + instance_id, + vmm_id, + small_resource_request(), + SledReservationConstraintBuilder::new().build(), + ) + .await + .expect("Failed to create reservation"); + vmm_id +} + +async fn delete_reservation( + opctx: &OpContext, + db: &DataStore, + vmm_id: PropolisUuid, +) { + db.sled_reservation_delete(&opctx, vmm_id) + .await + .expect("Failed to delete reservation"); +} + +///////////////////////////////////////////////////////////////// +// +// TEST HARNESS +// +// This structure shares logic between benchmarks, making it easy +// to perform shared tasks such as creating contention for reservations. + +struct TestHarness { + log: Logger, + db: TestDatabase, + sleds: Vec, +} + +impl TestHarness { + async fn new(log: &Logger, sled_count: usize) -> Self { + let db = TestDatabase::new_with_datastore(log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + let sleds = create_sleds(&datastore, sled_count).await; + + Self { log: log.clone(), db, sleds } + } + + async fn create_reservation(&self) -> PropolisUuid { + let (opctx, datastore) = (self.db.opctx(), self.db.datastore()); + create_reservation(opctx, datastore).await + } + + async fn delete_reservation(&self, vmm_id: PropolisUuid) { + let (opctx, datastore) = (self.db.opctx(), self.db.datastore()); + delete_reservation(opctx, datastore, vmm_id).await + } + + // Spin up a number of background tasks which perform the work of "create + // reservation, destroy reservation" in a loop. + fn create_contention(&self, count: usize) -> ContendingTasks { + let mut tasks = tokio::task::JoinSet::new(); + let should_exit = Arc::new(AtomicBool::new(false)); + + for _ in 0..count { + tasks.spawn({ + let should_exit = should_exit.clone(); + let opctx = + self.db.opctx().child(std::collections::BTreeMap::new()); + let datastore = self.db.datastore().clone(); + async move { + loop { + if should_exit.load(Ordering::SeqCst) { + return; + } + + let vmm_id = + create_reservation(&opctx, &datastore).await; + delete_reservation(&opctx, &datastore, vmm_id).await; + } + } + }); + } + + ContendingTasks { tasks, should_exit } + } + + async fn terminate(self) { + self.db.terminate().await; + } +} + +// A handle to tasks created by [TestHarness::create_contention]. +// +// Should be terminated after the benchmark has completed executing. +#[must_use] +struct ContendingTasks { + tasks: tokio::task::JoinSet<()>, + should_exit: Arc, +} + +impl ContendingTasks { + async fn terminate(self) { + self.should_exit.store(true, Ordering::SeqCst); + self.tasks.join_all().await; + } +} + +///////////////////////////////////////////////////////////////// +// +// PARAMETERS +// +// Describes varations between otherwise shared test logic + +#[derive(Copy, Clone)] +struct TestParams { + vmms: usize, + contending_tasks: usize, +} + +///////////////////////////////////////////////////////////////// +// +// BENCHMARKS +// +// You can run these with the following command: +// +// ```bash +// cargo bench -p nexus-db-queries +// ``` + +async fn bench_reservation( + log: &Logger, + params: TestParams, + iterations: u64, +) -> Duration { + const SLED_COUNT: usize = 4; + let harness = TestHarness::new(&log, SLED_COUNT).await; + let tasks = harness.create_contention(params.contending_tasks); + + let start = Instant::now(); + for _ in 0..iterations { + let mut vmm_ids = Vec::with_capacity(params.vmms); + black_box({ + for _ in 0..params.vmms { + vmm_ids.push(harness.create_reservation().await); + } + + for vmm_id in vmm_ids.drain(..) { + harness.delete_reservation(vmm_id).await; + } + }); + } + let duration = start.elapsed(); + + tasks.terminate().await; + harness.terminate().await; + duration +} + +// Typically we run our database tests using "cargo nextest run", +// which triggers the "crdb-seed" binary to create an initialized +// database when we boot up. +// +// If we're using "cargo bench", we don't get that guarantee. +// Go through the database ensuring process manually. +async fn setup_db(log: &Logger) { + print!("setting up seed cockroachdb database... "); + let (seed_tar, status) = dev::seed::ensure_seed_tarball_exists( + log, + dev::seed::should_invalidate_seed(), + ) + .await + .expect("Failed to create seed tarball for CRDB"); + status.log(log, &seed_tar); + unsafe { + std::env::set_var(dev::CRDB_SEED_TAR_ENV, seed_tar); + } + println!("OK"); +} + +fn sled_reservation_benchmark(c: &mut Criterion) { + let logctx = dev::test_setup_log("sled-reservation"); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(setup_db(&logctx.log)); + + let mut group = c.benchmark_group("vmm-reservation"); + for vmms in [1, 10, 100] { + for contending_tasks in [0, 1, 2] { + let params = TestParams { vmms, contending_tasks }; + let name = format!( + "{vmms}-vmms-{contending_tasks}-other-tasks" + ); + + group.bench_function(&name, |b| { + b.to_async(&rt).iter_custom(|iters| { + let log = logctx.log.clone(); + async move { + bench_reservation(&log, params, iters).await + } + }) + }); + } + } + group.finish(); + + logctx.cleanup_successful(); +} + +criterion_group!( + name = benches; + // To accomodate the fact that these benchmarks are a bit bulky, + // we set the following: + // - Smaller sample size, to keep running time down + // - Higher noise threshold, to avoid avoid false positive change detection + config = Criterion::default() + .sample_size(10) + .noise_threshold(0.10); + targets = sled_reservation_benchmark +); +criterion_main!(benches); From 04a4b982922c2c19f7ef2572b2229c86c98a8f1f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Feb 2025 14:15:05 -0800 Subject: [PATCH 29/84] Tweak usable hardware threads to make instance placement less flaky --- nexus/db-queries/benches/sled_reservation_benchmark.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs index 4d9c8bae717..de52e66a469 100644 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -82,7 +82,7 @@ fn sled_baseboard_for_test() -> SledBaseboard { fn sled_system_hardware_for_test() -> SledSystemHardware { SledSystemHardware { is_scrimlet: false, - usable_hardware_threads: 4, + usable_hardware_threads: 32, usable_physical_ram: ByteCount::try_from(1 << 40).unwrap(), reservoir_size: ByteCount::try_from(1 << 39).unwrap(), } From db40d058efaad3fbd0e58c2818a40ac7793482ec Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Feb 2025 14:44:02 -0800 Subject: [PATCH 30/84] Normalize reservation time, only benchmark creation pathway --- .../benches/sled_reservation_benchmark.rs | 86 ++++++++++++------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs index de52e66a469..291ac4325a3 100644 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -155,20 +155,18 @@ async fn delete_reservation( // to perform shared tasks such as creating contention for reservations. struct TestHarness { - log: Logger, db: TestDatabase, - sleds: Vec, } impl TestHarness { async fn new(log: &Logger, sled_count: usize) -> Self { let db = TestDatabase::new_with_datastore(log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let (authz_project, _project) = + let (_authz_project, _project) = create_project(&opctx, &datastore).await; - let sleds = create_sleds(&datastore, sled_count).await; + create_sleds(&datastore, sled_count).await; - Self { log: log.clone(), db, sleds } + Self { db } } async fn create_reservation(&self) -> PropolisUuid { @@ -185,17 +183,17 @@ impl TestHarness { // reservation, destroy reservation" in a loop. fn create_contention(&self, count: usize) -> ContendingTasks { let mut tasks = tokio::task::JoinSet::new(); - let should_exit = Arc::new(AtomicBool::new(false)); + let is_terminating = Arc::new(AtomicBool::new(false)); for _ in 0..count { tasks.spawn({ - let should_exit = should_exit.clone(); + let is_terminating = is_terminating.clone(); let opctx = self.db.opctx().child(std::collections::BTreeMap::new()); let datastore = self.db.datastore().clone(); async move { loop { - if should_exit.load(Ordering::SeqCst) { + if is_terminating.load(Ordering::SeqCst) { return; } @@ -207,7 +205,7 @@ impl TestHarness { }); } - ContendingTasks { tasks, should_exit } + ContendingTasks { tasks, is_terminating } } async fn terminate(self) { @@ -221,12 +219,12 @@ impl TestHarness { #[must_use] struct ContendingTasks { tasks: tokio::task::JoinSet<()>, - should_exit: Arc, + is_terminating: Arc, } impl ContendingTasks { async fn terminate(self) { - self.should_exit.store(true, Ordering::SeqCst); + self.is_terminating.store(true, Ordering::SeqCst); self.tasks.join_all().await; } } @@ -239,10 +237,14 @@ impl ContendingTasks { #[derive(Copy, Clone)] struct TestParams { + // Number of VMMs to provision from the task-under-test vmms: usize, contending_tasks: usize, } +const VMM_PARAMS: [usize; 3] = [1, 10, 100]; +const TASK_PARAMS: [usize; 3] = [0, 1, 2]; + ///////////////////////////////////////////////////////////////// // // BENCHMARKS @@ -261,21 +263,49 @@ async fn bench_reservation( const SLED_COUNT: usize = 4; let harness = TestHarness::new(&log, SLED_COUNT).await; let tasks = harness.create_contention(params.contending_tasks); + let mut vmm_ids = Vec::with_capacity(params.vmms); + + let duration = { + let mut total_duration = Duration::ZERO; + for _ in 0..iterations { + let start = Instant::now(); + + // Clippy: We don't want to move this block outside of "black_box", even though it + // isn't returning anything. That would defeat the whole point of using "black_box", + // which is to avoid profiling code that is optimized based on the surrounding + // benchmark function. + #[allow(clippy::unit_arg)] + black_box({ + // Create all the requested vmms. + // + // Note that all prior reservations will remain in the DB as we continue + // provisioning the "next" VMM. + for _ in 0..params.vmms { + vmm_ids.push(harness.create_reservation().await); + } + }); + let iter_duration = start.elapsed(); + + // Return the "average time to provision a single VMM". + // + // This normalizes the results, regardless of how many VMMs we are provisioning. + // + // Note that we expect additional contention to create more work, but it's difficult to + // normalize "how much work is being created by contention". + total_duration += std::time::Duration::from_nanos( + u64::try_from(iter_duration.as_nanos() / params.vmms as u128) + .expect("This benchmark is taking hundreds of years to run, maybe optimize it") + ); - let start = Instant::now(); - for _ in 0..iterations { - let mut vmm_ids = Vec::with_capacity(params.vmms); - black_box({ - for _ in 0..params.vmms { - vmm_ids.push(harness.create_reservation().await); - } - + // Clean up all our VMMs. + // + // We don't really care how long this takes, so we omit it from the tracking time. for vmm_id in vmm_ids.drain(..) { harness.delete_reservation(vmm_id).await; } - }); - } - let duration = start.elapsed(); + } + total_duration + }; tasks.terminate().await; harness.terminate().await; @@ -310,19 +340,15 @@ fn sled_reservation_benchmark(c: &mut Criterion) { rt.block_on(setup_db(&logctx.log)); let mut group = c.benchmark_group("vmm-reservation"); - for vmms in [1, 10, 100] { - for contending_tasks in [0, 1, 2] { + for vmms in VMM_PARAMS { + for contending_tasks in TASK_PARAMS { let params = TestParams { vmms, contending_tasks }; - let name = format!( - "{vmms}-vmms-{contending_tasks}-other-tasks" - ); + let name = format!("{vmms}-vmms-{contending_tasks}-other-tasks"); group.bench_function(&name, |b| { b.to_async(&rt).iter_custom(|iters| { let log = logctx.log.clone(); - async move { - bench_reservation(&log, params, iters).await - } + async move { bench_reservation(&log, params, iters).await } }) }); } From b68239b998758459ab78885c0ab660704a150d70 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Feb 2025 16:26:54 -0800 Subject: [PATCH 31/84] Normalize --- .../benches/sled_reservation_benchmark.rs | 175 ++++++++++++------ nexus/db-queries/src/transaction_retry.rs | 2 +- 2 files changed, 119 insertions(+), 58 deletions(-) diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs index 291ac4325a3..9912e4b351e 100644 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -4,6 +4,8 @@ //! Benchmarks creating sled reservations +use anyhow::Context; +use anyhow::Result; use criterion::black_box; use criterion::{criterion_group, criterion_main, Criterion}; use nexus_db_model::ByteCount; @@ -122,7 +124,7 @@ fn small_resource_request() -> Resources { ) } -async fn create_reservation(opctx: &OpContext, db: &DataStore) -> PropolisUuid { +async fn create_reservation(opctx: &OpContext, db: &DataStore) -> Result { let instance_id = InstanceUuid::new_v4(); let vmm_id = PropolisUuid::new_v4(); db.sled_reservation_create( @@ -133,18 +135,19 @@ async fn create_reservation(opctx: &OpContext, db: &DataStore) -> PropolisUuid { SledReservationConstraintBuilder::new().build(), ) .await - .expect("Failed to create reservation"); - vmm_id + .context("Failed to create reservation")?; + Ok(vmm_id) } async fn delete_reservation( opctx: &OpContext, db: &DataStore, vmm_id: PropolisUuid, -) { +) -> Result<()> { db.sled_reservation_delete(&opctx, vmm_id) .await - .expect("Failed to delete reservation"); + .context("Failed to delete reservation")?; + Ok(()) } ///////////////////////////////////////////////////////////////// @@ -169,18 +172,11 @@ impl TestHarness { Self { db } } - async fn create_reservation(&self) -> PropolisUuid { - let (opctx, datastore) = (self.db.opctx(), self.db.datastore()); - create_reservation(opctx, datastore).await - } - - async fn delete_reservation(&self, vmm_id: PropolisUuid) { - let (opctx, datastore) = (self.db.opctx(), self.db.datastore()); - delete_reservation(opctx, datastore, vmm_id).await - } - // Spin up a number of background tasks which perform the work of "create // reservation, destroy reservation" in a loop. + // + // TODO: Have all of these tasks call "bench_one_iter", and return + // the total latency. Then divide by the number of workers. fn create_contention(&self, count: usize) -> ContendingTasks { let mut tasks = tokio::task::JoinSet::new(); let is_terminating = Arc::new(AtomicBool::new(false)); @@ -197,9 +193,12 @@ impl TestHarness { return; } - let vmm_id = - create_reservation(&opctx, &datastore).await; - delete_reservation(&opctx, &datastore, vmm_id).await; + let vmm_id = create_reservation(&opctx, &datastore) + .await + .expect("Task causing contention failed to reserve"); + delete_reservation(&opctx, &datastore, vmm_id) + .await + .expect("Task causing contention failed to delete"); } } }); @@ -237,13 +236,13 @@ impl ContendingTasks { #[derive(Copy, Clone)] struct TestParams { - // Number of VMMs to provision from the task-under-test + // Number of vmms to provision from the task-under-test vmms: usize, - contending_tasks: usize, + tasks: usize, } const VMM_PARAMS: [usize; 3] = [1, 10, 100]; -const TASK_PARAMS: [usize; 3] = [0, 1, 2]; +const TASK_PARAMS: [usize; 3] = [1, 2, 3]; ///////////////////////////////////////////////////////////////// // @@ -255,6 +254,63 @@ const TASK_PARAMS: [usize; 3] = [0, 1, 2]; // cargo bench -p nexus-db-queries // ``` +// Average a duration over a divisor. +// +// For example, if we reserve 100 vmms, you can use "100" as the divisor +// to get the "average duration to provision a vmm". +fn average_duration(duration: Duration, divisor: usize) -> Duration { + assert_ne!(divisor, 0, "Don't divide by zero please"); + + Duration::from_nanos( + u64::try_from(duration.as_nanos() / divisor as u128) + .expect("This benchmark is taking hundreds of years to run, maybe optimize it") + ) +} + +// Reserves "params.vmms" vmms, and later deletes their reservations. +// +// Returns the average time to provision a single vmm. +async fn reserve_vmms_and_return_average_duration( + params: &TestParams, + opctx: &OpContext, + db: &DataStore, +) -> Duration { + let mut vmm_ids = Vec::with_capacity(params.vmms); + let start = Instant::now(); + + // Clippy: We don't want to move this block outside of "black_box", even though it + // isn't returning anything. That would defeat the whole point of using "black_box", + // which is to avoid profiling code that is optimized based on the surrounding + // benchmark function. + #[allow(clippy::unit_arg)] + black_box({ + // Create all the requested vmms. + // + // Note that all prior reservations will remain in the DB as we continue + // provisioning the "next" vmm. + for _ in 0..params.vmms { + vmm_ids.push(create_reservation(opctx, db).await.expect("Failed to provision vmm")); + } + }); + + // Return the "average time to provision a single vmm". + // + // This normalizes the results, regardless of how many vmms we are provisioning. + // + // Note that we expect additional contention to create more work, but it's difficult to + // normalize "how much work is being created by contention". + let duration = average_duration(start.elapsed(), params.vmms); + + // Clean up all our vmms. + // + // We don't really care how long this takes, so we omit it from the tracking time. + for vmm_id in vmm_ids.drain(..) { + delete_reservation(opctx, db,vmm_id).await.expect("Failed to delete vmm"); + } + + duration +} + async fn bench_reservation( log: &Logger, params: TestParams, @@ -262,47 +318,52 @@ async fn bench_reservation( ) -> Duration { const SLED_COUNT: usize = 4; let harness = TestHarness::new(&log, SLED_COUNT).await; - let tasks = harness.create_contention(params.contending_tasks); - let mut vmm_ids = Vec::with_capacity(params.vmms); + let tasks = harness.create_contention(params.tasks); let duration = { let mut total_duration = Duration::ZERO; + + // Each iteration is an "attempt" at the test. for _ in 0..iterations { - let start = Instant::now(); - - // Clippy: We don't want to move this block outside of "black_box", even though it - // isn't returning anything. That would defeat the whole point of using "black_box", - // which is to avoid profiling code that is optimized based on the surrounding - // benchmark function. - #[allow(clippy::unit_arg)] - black_box({ - // Create all the requested vmms. - // - // Note that all prior reservations will remain in the DB as we continue - // provisioning the "next" VMM. - for _ in 0..params.vmms { - vmm_ids.push(harness.create_reservation().await); - } - }); - let iter_duration = start.elapsed(); + // Within each attempt, we spawn the tasks requested. + let mut set = tokio::task::JoinSet::new(); + for _ in 0..params.tasks { + set.spawn({ + let opctx = harness.db.opctx().child(std::collections::BTreeMap::new()); + let db = harness.db.datastore().clone(); + async move { + reserve_vmms_and_return_average_duration(¶ms, &opctx, &db).await + } + }); + } + + // The sum of "average time to provision a single vmm" across all tasks. + let all_tasks_duration = set.join_all().await.into_iter().fold(Duration::ZERO, |acc, x| acc + x); - // Return the "average time to provision a single VMM". + // The normalized "time to provision a single vmm", across both: + // - The number of vmms reserved by each task, and + // - The number of tasks // - // This normalizes the results, regardless of how many VMMs we are provisioning. + // As an example, if we provision 10 vmms, and have 5 tasks, and we assume + // that VM provisioning time is exactly one second (contention has no impact, somehow): // - // Note that we expect additional contention to create more work, but it's difficult to - // normalize "how much work is being created by contention". - total_duration += std::time::Duration::from_nanos( - u64::try_from(iter_duration.as_nanos() / params.vmms as u128) - .expect("This benchmark is taking hundreds of years to run, maybe optimize it") - ); - - // Clean up all our VMMs. + // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average + // duration of "1 second". + // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds + // (1 second average * 5 tasks). + // - So, we'd increment our "total_duration" by "1 second per vmm", which has been + // normalized cross both the tasks and quantities of vmms. // - // We don't really care how long this takes, so we omit it from the tracking time. - for vmm_id in vmm_ids.drain(..) { - harness.delete_reservation(vmm_id).await; - } + // Why bother doing this? + // + // When we perform this normalization, we can vary the "total vmms provisioned" as well + // as "total tasks" significantly, but easily compare test durations with one another. + // + // For example: if the total number of vmms has no impact on the next provisioning + // request, we should see similar durations for "100 vmms reserved" vs "1 vmm + // reserved". However, if more vmms actually make reservation times slower, we'll see + // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: + total_duration += average_duration(all_tasks_duration, params.tasks); } total_duration }; @@ -341,9 +402,9 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("vmm-reservation"); for vmms in VMM_PARAMS { - for contending_tasks in TASK_PARAMS { - let params = TestParams { vmms, contending_tasks }; - let name = format!("{vmms}-vmms-{contending_tasks}-other-tasks"); + for tasks in TASK_PARAMS { + let params = TestParams { vmms, tasks }; + let name = format!("{vmms}-vmms-{tasks}-tasks"); group.bench_function(&name, |b| { b.to_async(&rt).iter_custom(|iters| { diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs index cf8ee223767..41b06019924 100644 --- a/nexus/db-queries/src/transaction_retry.rs +++ b/nexus/db-queries/src/transaction_retry.rs @@ -75,7 +75,7 @@ pub struct RetryHelper { const MIN_RETRY_BACKOFF: Duration = Duration::from_millis(0); const MAX_RETRY_BACKOFF: Duration = Duration::from_millis(50); -const MAX_RETRY_ATTEMPTS: u32 = 10; +const MAX_RETRY_ATTEMPTS: u32 = 100; impl RetryHelper { /// Creates a new RetryHelper, and starts a timer tracking the transaction From ca8f8902cb96ac89962eae985c89c4ceb644a804 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 7 Feb 2025 00:43:29 -0800 Subject: [PATCH 32/84] cleanup --- .../benches/sled_reservation_benchmark.rs | 104 +++++++----------- 1 file changed, 41 insertions(+), 63 deletions(-) diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs index 9912e4b351e..96a8299ed1e 100644 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -29,11 +29,10 @@ use omicron_uuid_kinds::PropolisUuid; use slog::Logger; use std::net::Ipv6Addr; use std::net::SocketAddrV6; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use tokio::sync::Barrier; use uuid::Uuid; ///////////////////////////////////////////////////////////////// @@ -124,7 +123,10 @@ fn small_resource_request() -> Resources { ) } -async fn create_reservation(opctx: &OpContext, db: &DataStore) -> Result { +async fn create_reservation( + opctx: &OpContext, + db: &DataStore, +) -> Result { let instance_id = InstanceUuid::new_v4(); let vmm_id = PropolisUuid::new_v4(); db.sled_reservation_create( @@ -172,62 +174,11 @@ impl TestHarness { Self { db } } - // Spin up a number of background tasks which perform the work of "create - // reservation, destroy reservation" in a loop. - // - // TODO: Have all of these tasks call "bench_one_iter", and return - // the total latency. Then divide by the number of workers. - fn create_contention(&self, count: usize) -> ContendingTasks { - let mut tasks = tokio::task::JoinSet::new(); - let is_terminating = Arc::new(AtomicBool::new(false)); - - for _ in 0..count { - tasks.spawn({ - let is_terminating = is_terminating.clone(); - let opctx = - self.db.opctx().child(std::collections::BTreeMap::new()); - let datastore = self.db.datastore().clone(); - async move { - loop { - if is_terminating.load(Ordering::SeqCst) { - return; - } - - let vmm_id = create_reservation(&opctx, &datastore) - .await - .expect("Task causing contention failed to reserve"); - delete_reservation(&opctx, &datastore, vmm_id) - .await - .expect("Task causing contention failed to delete"); - } - } - }); - } - - ContendingTasks { tasks, is_terminating } - } - async fn terminate(self) { self.db.terminate().await; } } -// A handle to tasks created by [TestHarness::create_contention]. -// -// Should be terminated after the benchmark has completed executing. -#[must_use] -struct ContendingTasks { - tasks: tokio::task::JoinSet<()>, - is_terminating: Arc, -} - -impl ContendingTasks { - async fn terminate(self) { - self.is_terminating.store(true, Ordering::SeqCst); - self.tasks.join_all().await; - } -} - ///////////////////////////////////////////////////////////////// // // PARAMETERS @@ -241,7 +192,7 @@ struct TestParams { tasks: usize, } -const VMM_PARAMS: [usize; 3] = [1, 10, 100]; +const VMM_PARAMS: [usize; 3] = [1, 8, 16]; const TASK_PARAMS: [usize; 3] = [1, 2, 3]; ///////////////////////////////////////////////////////////////// @@ -289,7 +240,11 @@ async fn reserve_vmms_and_return_average_duration( // Note that all prior reservations will remain in the DB as we continue // provisioning the "next" vmm. for _ in 0..params.vmms { - vmm_ids.push(create_reservation(opctx, db).await.expect("Failed to provision vmm")); + vmm_ids.push( + create_reservation(opctx, db) + .await + .expect("Failed to provision vmm"), + ); } }); @@ -305,7 +260,9 @@ async fn reserve_vmms_and_return_average_duration( // // We don't really care how long this takes, so we omit it from the tracking time. for vmm_id in vmm_ids.drain(..) { - delete_reservation(opctx, db,vmm_id).await.expect("Failed to delete vmm"); + delete_reservation(opctx, db, vmm_id) + .await + .expect("Failed to delete vmm"); } duration @@ -318,7 +275,6 @@ async fn bench_reservation( ) -> Duration { const SLED_COUNT: usize = 4; let harness = TestHarness::new(&log, SLED_COUNT).await; - let tasks = harness.create_contention(params.tasks); let duration = { let mut total_duration = Duration::ZERO; @@ -327,18 +283,40 @@ async fn bench_reservation( for _ in 0..iterations { // Within each attempt, we spawn the tasks requested. let mut set = tokio::task::JoinSet::new(); + + // This barrier exists to lessen the impact of "task spawning" on the benchmark. + // + // We want to have all tasks run as concurrently as possible, since we're trying to + // measure contention explicitly. + let barrier = Arc::new(Barrier::new(params.tasks)); + for _ in 0..params.tasks { set.spawn({ - let opctx = harness.db.opctx().child(std::collections::BTreeMap::new()); + let opctx = harness + .db + .opctx() + .child(std::collections::BTreeMap::new()); let db = harness.db.datastore().clone(); + let barrier = barrier.clone(); async move { - reserve_vmms_and_return_average_duration(¶ms, &opctx, &db).await + // Wait until all tasks are ready... + barrier.wait().await; + + // ... and then actually do the benchmark + reserve_vmms_and_return_average_duration( + ¶ms, &opctx, &db, + ) + .await } }); } // The sum of "average time to provision a single vmm" across all tasks. - let all_tasks_duration = set.join_all().await.into_iter().fold(Duration::ZERO, |acc, x| acc + x); + let all_tasks_duration = set + .join_all() + .await + .into_iter() + .fold(Duration::ZERO, |acc, x| acc + x); // The normalized "time to provision a single vmm", across both: // - The number of vmms reserved by each task, and @@ -363,12 +341,12 @@ async fn bench_reservation( // request, we should see similar durations for "100 vmms reserved" vs "1 vmm // reserved". However, if more vmms actually make reservation times slower, we'll see // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: - total_duration += average_duration(all_tasks_duration, params.tasks); + total_duration += + average_duration(all_tasks_duration, params.tasks); } total_duration }; - tasks.terminate().await; harness.terminate().await; duration } From 5406dd489c969852c2f8c34ac34ba1b2f6aae55b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 10 Feb 2025 12:54:07 -0800 Subject: [PATCH 33/84] Better contention info --- Cargo.lock | 2 + .../benches/sled_reservation_benchmark.rs | 134 ++++++++-- nexus/db-queries/src/transaction_retry.rs | 2 +- nexus/test-utils/Cargo.toml | 2 + nexus/test-utils/src/lib.rs | 1 + nexus/test-utils/src/sql.rs | 239 ++++++++++++++++++ nexus/tests/integration_tests/schema.rs | 228 ++--------------- 7 files changed, 379 insertions(+), 229 deletions(-) create mode 100644 nexus/test-utils/src/sql.rs diff --git a/Cargo.lock b/Cargo.lock index a1e4dcc1f9d..7985ed91e82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6324,12 +6324,14 @@ dependencies = [ "oximeter-collector", "oximeter-producer", "oxnet", + "pretty_assertions", "serde", "serde_json", "serde_urlencoded", "sled-agent-client", "slog", "tokio", + "tokio-postgres", "tokio-util", "uuid", ] diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs index 96a8299ed1e..3449fb68946 100644 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -21,12 +21,15 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DataStore; +use nexus_test_utils::sql::process_rows; +use nexus_test_utils::sql::Row; use nexus_types::external_api::params; use omicron_common::api::external; use omicron_test_utils::dev; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use slog::Logger; +use std::collections::HashMap; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use std::sync::Arc; @@ -174,6 +177,89 @@ impl TestHarness { Self { db } } + // Emit internal CockroachDb information about contention + async fn print_contention(&self) { + let client = self.db.crdb().connect().await.expect("Failed to connect to db"); + + let queries = [ + "SELECT table_name, index_name, num_contention_events::TEXT FROM crdb_internal.cluster_contended_indexes", + "SELECT table_name,num_contention_events::TEXT FROM crdb_internal.cluster_contended_tables", + "WITH c AS (SELECT DISTINCT ON (table_id, index_id) table_id, index_id, num_contention_events AS events, cumulative_contention_time AS time FROM crdb_internal.cluster_contention_events) SELECT i.descriptor_name, i.index_name, c.events::TEXT, c.time::TEXT FROM crdb_internal.table_indexes AS i JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id ORDER BY c.time DESC LIMIT 10;" + ]; + + // Used for padding: get a map of "column name" -> "max value length". + let max_lengths_by_column = |rows: &Vec| { + let mut lengths = HashMap::new(); + for row in rows { + for column in &row.values { + let value_len = column.value().unwrap().as_str().len(); + let name_len = column.name().len(); + let len = std::cmp::max(value_len, name_len); + + lengths.entry(column.name().to_string()) + .and_modify(|entry| { + if len > *entry { + *entry = len; + } + }) + .or_insert(len); + } + } + lengths + }; + + for sql in queries { + let rows = client.query(sql, &[]) + .await.expect("Failed to query contended tables"); + let rows = process_rows(&rows); + if rows.is_empty() { + continue; + } + + println!("{sql}"); + let max_lengths = max_lengths_by_column(&rows); + let mut header = true; + + for row in rows { + if header { + let mut total_len = 0; + for column in &row.values { + let width = max_lengths.get(column.name()).unwrap(); + print!(" {:width$} ", column.name()); + print!("|"); + total_len += width + 3; + } + println!(""); + println!("{:- Arc { + Arc::new( + self.db + .opctx() + .child(std::collections::BTreeMap::new()) + ) + } + + fn db(&self) -> Arc { + self.db.datastore().clone() + } + async fn terminate(self) { self.db.terminate().await; } @@ -192,8 +278,10 @@ struct TestParams { tasks: usize, } -const VMM_PARAMS: [usize; 3] = [1, 8, 16]; -const TASK_PARAMS: [usize; 3] = [1, 2, 3]; +// const VMM_PARAMS: [usize; 3] = [1, 8, 16]; +// const TASK_PARAMS: [usize; 3] = [1, 2, 3]; +const VMM_PARAMS: [usize; 1] = [4]; +const TASK_PARAMS: [usize; 1] = [4]; ///////////////////////////////////////////////////////////////// // @@ -269,13 +357,11 @@ async fn reserve_vmms_and_return_average_duration( } async fn bench_reservation( - log: &Logger, + opctx: Arc, + db: Arc, params: TestParams, iterations: u64, ) -> Duration { - const SLED_COUNT: usize = 4; - let harness = TestHarness::new(&log, SLED_COUNT).await; - let duration = { let mut total_duration = Duration::ZERO; @@ -292,12 +378,10 @@ async fn bench_reservation( for _ in 0..params.tasks { set.spawn({ - let opctx = harness - .db - .opctx() - .child(std::collections::BTreeMap::new()); - let db = harness.db.datastore().clone(); + let opctx = opctx.clone(); + let db = db.clone(); let barrier = barrier.clone(); + async move { // Wait until all tasks are ready... barrier.wait().await; @@ -347,7 +431,6 @@ async fn bench_reservation( total_duration }; - harness.terminate().await; duration } @@ -384,16 +467,37 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let params = TestParams { vmms, tasks }; let name = format!("{vmms}-vmms-{tasks}-tasks"); + // Initialize the harness before calling "bench_function" so + // that the "warm-up" calls to "bench_function" are actually useful + // at warming up the database. + // + // This mitigates any database-caching issues like "loading schema + // on boot", or "connection pooling", as the pool stays the same + // between calls to the benchmark function. + let log = logctx.log.clone(); + let harness = rt.block_on(async move { + const SLED_COUNT: usize = 4; + TestHarness::new(&log, SLED_COUNT).await + }); + + // Actually invoke the benchmark. group.bench_function(&name, |b| { b.to_async(&rt).iter_custom(|iters| { - let log = logctx.log.clone(); - async move { bench_reservation(&log, params, iters).await } + let opctx = harness.opctx(); + let db = harness.db(); + async move { bench_reservation(opctx, db, params, iters).await } }) }); + + // Clean-up the harness; we'll use a new database between + // varations in parameters. + rt.block_on(async move { + harness.print_contention().await; + harness.terminate().await; + }); } } group.finish(); - logctx.cleanup_successful(); } diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs index 41b06019924..f626fcc80cd 100644 --- a/nexus/db-queries/src/transaction_retry.rs +++ b/nexus/db-queries/src/transaction_retry.rs @@ -75,7 +75,7 @@ pub struct RetryHelper { const MIN_RETRY_BACKOFF: Duration = Duration::from_millis(0); const MAX_RETRY_BACKOFF: Duration = Duration::from_millis(50); -const MAX_RETRY_ATTEMPTS: u32 = 100; +const MAX_RETRY_ATTEMPTS: u32 = 50; impl RetryHelper { /// Creates a new RetryHelper, and starts a timer tracking the transaction diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 3756c6f86a6..21d490e1365 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -42,12 +42,14 @@ oximeter.workspace = true oximeter-collector.workspace = true oximeter-producer.workspace = true oxnet.workspace = true +pretty_assertions.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded.workspace = true sled-agent-client.workspace = true slog.workspace = true tokio.workspace = true +tokio-postgres = { workspace = true, features = ["with-serde_json-1"] } tokio-util.workspace = true hickory-resolver.workspace = true uuid.workspace = true diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 6d35b5e4943..1ba16c9b73a 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -103,6 +103,7 @@ pub mod background; pub mod db; pub mod http_testing; pub mod resource_helpers; +pub mod sql; pub const SLED_AGENT_UUID: &str = "b6d65341-167c-41df-9b5c-41cded99c229"; pub const SLED_AGENT2_UUID: &str = "039be560-54cc-49e3-88df-1a29dadbf913"; diff --git a/nexus/test-utils/src/sql.rs b/nexus/test-utils/src/sql.rs new file mode 100644 index 00000000000..bdc4b443475 --- /dev/null +++ b/nexus/test-utils/src/sql.rs @@ -0,0 +1,239 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utilites to query "any data" from the database. +//! +//! This is useful for parsing internal cockroach structures and schemas, but +//! the typing is dynamic. As a result, this is relegated to a test-only +//! crate. + +use chrono::{DateTime, Utc}; +use pretty_assertions::assert_eq; +use std::net::IpAddr; +use uuid::Uuid; + +/// A named SQL enum, of the form "typename, value" +#[derive(PartialEq, Clone, Debug)] +pub struct SqlEnum { + name: String, + variant: String, +} + +impl From<(&str, &str)> for SqlEnum { + fn from((name, variant): (&str, &str)) -> Self { + Self { name: name.to_string(), variant: variant.to_string() } + } +} + +/// A newtype wrapper around a string, which allows us to more liberally +/// interpret SQL types. +/// +/// Note that for the purposes of schema comparisons, we don't care about parsing +/// the contents of the database, merely the schema and equality of contained +/// data. +#[derive(PartialEq, Clone, Debug)] +pub enum AnySqlType { + Bool(bool), + DateTime, + Enum(SqlEnum), + Float4(f32), + Int4(i32), + Int8(i64), + Json(serde_json::value::Value), + String(String), + TextArray(Vec), + Uuid(Uuid), + Inet(IpAddr), + // TODO: This isn't exhaustive, feel free to add more. + // + // These should only be necessary for rows where the database schema changes also choose to + // populate data. +} + +impl From for AnySqlType { + fn from(b: bool) -> Self { + Self::Bool(b) + } +} + +impl From for AnySqlType { + fn from(value: SqlEnum) -> Self { + Self::Enum(value) + } +} + +impl From for AnySqlType { + fn from(value: f32) -> Self { + Self::Float4(value) + } +} + +impl From for AnySqlType { + fn from(value: i32) -> Self { + Self::Int4(value) + } +} + +impl From for AnySqlType { + fn from(value: i64) -> Self { + Self::Int8(value) + } +} + +impl From for AnySqlType { + fn from(value: String) -> Self { + Self::String(value) + } +} + +impl From for AnySqlType { + fn from(value: Uuid) -> Self { + Self::Uuid(value) + } +} + +impl From for AnySqlType { + fn from(value: IpAddr) -> Self { + Self::Inet(value) + } +} + +impl AnySqlType { + pub fn as_str(&self) -> &str { + match self { + AnySqlType::String(s) => s, + _ => panic!("Not a string type"), + } + } +} + +impl<'a> tokio_postgres::types::FromSql<'a> for AnySqlType { + fn from_sql( + ty: &tokio_postgres::types::Type, + raw: &'a [u8], + ) -> Result> { + if String::accepts(ty) { + return Ok(AnySqlType::String(String::from_sql(ty, raw)?)); + } + if DateTime::::accepts(ty) { + // We intentionally drop the time here -- we only care that there + // is some value present. + let _ = DateTime::::from_sql(ty, raw)?; + return Ok(AnySqlType::DateTime); + } + if bool::accepts(ty) { + return Ok(AnySqlType::Bool(bool::from_sql(ty, raw)?)); + } + if Uuid::accepts(ty) { + return Ok(AnySqlType::Uuid(Uuid::from_sql(ty, raw)?)); + } + if i32::accepts(ty) { + return Ok(AnySqlType::Int4(i32::from_sql(ty, raw)?)); + } + if i64::accepts(ty) { + return Ok(AnySqlType::Int8(i64::from_sql(ty, raw)?)); + } + if f32::accepts(ty) { + return Ok(AnySqlType::Float4(f32::from_sql(ty, raw)?)); + } + if serde_json::value::Value::accepts(ty) { + return Ok(AnySqlType::Json(serde_json::value::Value::from_sql( + ty, raw, + )?)); + } + if Vec::::accepts(ty) { + return Ok(AnySqlType::TextArray(Vec::::from_sql( + ty, raw, + )?)); + } + if IpAddr::accepts(ty) { + return Ok(AnySqlType::Inet(IpAddr::from_sql(ty, raw)?)); + } + + use tokio_postgres::types::Kind; + match ty.kind() { + Kind::Enum(_) => { + Ok(AnySqlType::Enum(SqlEnum { + name: ty.name().to_string(), + variant: std::str::from_utf8(raw)?.to_string(), + })) + }, + _ => { + Err(anyhow::anyhow!( + "Cannot parse type {ty:?}. \ + If you're trying to use this type in a table which is populated \ + during a schema migration, consider adding it to `AnySqlType`." + ).into()) + } + } + } + + fn accepts(_ty: &tokio_postgres::types::Type) -> bool { + true + } +} + +// It's a little redunant to include the column name alongside each value, +// but it results in a prettier diff. +#[derive(PartialEq, Debug)] +pub struct ColumnValue { + column: String, + value: Option, +} + +impl ColumnValue { + /// Creates a new column with a non-null value of "value" + pub fn new>(column: &str, value: V) -> Self { + Self { column: String::from(column), value: Some(value.into()) } + } + + /// Creates a new column with a "NULL" value + pub fn null(column: &str) -> Self { + Self { column: String::from(column), value: None } + } + + /// Returns the name of the column + pub fn name(&self) -> &str { + &self.column + } + + /// Returns the value of the column + pub fn value(&self) -> Option<&AnySqlType> { + self.value.as_ref() + } + + /// Returns the value of the column, asserting the name of the column + pub fn expect(&self, column: &str) -> Option<&AnySqlType> { + assert_eq!(self.column, column); + self.value() + } +} + +/// A generic representation of a row of SQL data +#[derive(PartialEq, Debug)] +pub struct Row { + pub values: Vec, +} + +impl Row { + pub fn new() -> Self { + Self { values: vec![] } + } +} + +pub fn process_rows(rows: &Vec) -> Vec { + let mut result = vec![]; + for row in rows { + let mut row_result = Row::new(); + for i in 0..row.len() { + let column_name = row.columns()[i].name(); + row_result.values.push(ColumnValue { + column: column_name.to_string(), + value: row.get(i), + }); + } + result.push(row_result); + } + result +} diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 6a5ce8ed0a3..50520e0a0de 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -3,7 +3,6 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use camino::Utf8PathBuf; -use chrono::{DateTime, Utc}; use dropshot::test_util::LogContext; use futures::future::BoxFuture; use nexus_config::NexusConfig; @@ -14,6 +13,10 @@ use nexus_db_model::{AllSchemaVersions, SchemaVersion}; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DISALLOW_FULL_TABLE_SCAN_SQL; use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; +use nexus_test_utils::sql::ColumnValue; +use nexus_test_utils::sql::Row; +use nexus_test_utils::sql::SqlEnum; +use nexus_test_utils::sql::process_rows; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; @@ -181,218 +184,6 @@ async fn query_crdb_schema_version(crdb: &CockroachInstance) -> String { version } -#[derive(PartialEq, Clone, Debug)] -struct SqlEnum { - name: String, - variant: String, -} - -impl From<(&str, &str)> for SqlEnum { - fn from((name, variant): (&str, &str)) -> Self { - Self { name: name.to_string(), variant: variant.to_string() } - } -} - -// A newtype wrapper around a string, which allows us to more liberally -// interpret SQL types. -// -// Note that for the purposes of schema comparisons, we don't care about parsing -// the contents of the database, merely the schema and equality of contained -// data. -#[derive(PartialEq, Clone, Debug)] -enum AnySqlType { - Bool(bool), - DateTime, - Enum(SqlEnum), - Float4(f32), - Int8(i64), - Json(serde_json::value::Value), - String(String), - TextArray(Vec), - Uuid(Uuid), - Inet(IpAddr), - // TODO: This isn't exhaustive, feel free to add more. - // - // These should only be necessary for rows where the database schema changes also choose to - // populate data. -} - -impl From for AnySqlType { - fn from(b: bool) -> Self { - Self::Bool(b) - } -} - -impl From for AnySqlType { - fn from(value: SqlEnum) -> Self { - Self::Enum(value) - } -} - -impl From for AnySqlType { - fn from(value: f32) -> Self { - Self::Float4(value) - } -} - -impl From for AnySqlType { - fn from(value: i64) -> Self { - Self::Int8(value) - } -} - -impl From for AnySqlType { - fn from(value: String) -> Self { - Self::String(value) - } -} - -impl From for AnySqlType { - fn from(value: Uuid) -> Self { - Self::Uuid(value) - } -} - -impl From for AnySqlType { - fn from(value: IpAddr) -> Self { - Self::Inet(value) - } -} - -impl AnySqlType { - fn as_str(&self) -> &str { - match self { - AnySqlType::String(s) => s, - _ => panic!("Not a string type"), - } - } -} - -impl<'a> tokio_postgres::types::FromSql<'a> for AnySqlType { - fn from_sql( - ty: &tokio_postgres::types::Type, - raw: &'a [u8], - ) -> Result> { - if String::accepts(ty) { - return Ok(AnySqlType::String(String::from_sql(ty, raw)?)); - } - if DateTime::::accepts(ty) { - // We intentionally drop the time here -- we only care that there - // is some value present. - let _ = DateTime::::from_sql(ty, raw)?; - return Ok(AnySqlType::DateTime); - } - if bool::accepts(ty) { - return Ok(AnySqlType::Bool(bool::from_sql(ty, raw)?)); - } - if Uuid::accepts(ty) { - return Ok(AnySqlType::Uuid(Uuid::from_sql(ty, raw)?)); - } - if i64::accepts(ty) { - return Ok(AnySqlType::Int8(i64::from_sql(ty, raw)?)); - } - if f32::accepts(ty) { - return Ok(AnySqlType::Float4(f32::from_sql(ty, raw)?)); - } - if serde_json::value::Value::accepts(ty) { - return Ok(AnySqlType::Json(serde_json::value::Value::from_sql( - ty, raw, - )?)); - } - if Vec::::accepts(ty) { - return Ok(AnySqlType::TextArray(Vec::::from_sql( - ty, raw, - )?)); - } - if IpAddr::accepts(ty) { - return Ok(AnySqlType::Inet(IpAddr::from_sql(ty, raw)?)); - } - - use tokio_postgres::types::Kind; - match ty.kind() { - Kind::Enum(_) => { - Ok(AnySqlType::Enum(SqlEnum { - name: ty.name().to_string(), - variant: std::str::from_utf8(raw)?.to_string(), - })) - }, - _ => { - Err(anyhow::anyhow!( - "Cannot parse type {ty:?}. \ - If you're trying to use this type in a table which is populated \ - during a schema migration, consider adding it to `AnySqlType`." - ).into()) - } - } - } - - fn accepts(_ty: &tokio_postgres::types::Type) -> bool { - true - } -} - -// It's a little redunant to include the column name alongside each value, -// but it results in a prettier diff. -#[derive(PartialEq, Debug)] -struct ColumnValue { - column: String, - value: Option, -} - -impl ColumnValue { - fn new>(column: &str, value: V) -> Self { - Self { column: String::from(column), value: Some(value.into()) } - } - - fn null(column: &str) -> Self { - Self { column: String::from(column), value: None } - } - - fn expect(&self, column: &str) -> Option<&AnySqlType> { - assert_eq!(self.column, column); - self.value.as_ref() - } -} - -// A generic representation of a row of SQL data -#[derive(PartialEq, Debug)] -struct Row { - values: Vec, -} - -impl Row { - fn new() -> Self { - Self { values: vec![] } - } -} - -enum ColumnSelector<'a> { - ByName(&'a [&'static str]), - Star, -} - -impl<'a> From<&'a [&'static str]> for ColumnSelector<'a> { - fn from(columns: &'a [&'static str]) -> Self { - Self::ByName(columns) - } -} - -fn process_rows(rows: &Vec) -> Vec { - let mut result = vec![]; - for row in rows { - let mut row_result = Row::new(); - for i in 0..row.len() { - let column_name = row.columns()[i].name(); - row_result.values.push(ColumnValue { - column: column_name.to_string(), - value: row.get(i), - }); - } - result.push(row_result); - } - result -} - async fn crdb_show_constraints( crdb: &CockroachInstance, table: &str, @@ -409,6 +200,17 @@ async fn crdb_show_constraints( process_rows(&rows) } +enum ColumnSelector<'a> { + ByName(&'a [&'static str]), + Star, +} + +impl<'a> From<&'a [&'static str]> for ColumnSelector<'a> { + fn from(columns: &'a [&'static str]) -> Self { + Self::ByName(columns) + } +} + async fn crdb_select( crdb: &CockroachInstance, columns: ColumnSelector<'_>, From bb0f3498149cb9dc4b8faec3e3376dfd75af9d45 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 11 Feb 2025 10:30:28 -0800 Subject: [PATCH 34/84] Better contention information --- .../benches/sled_reservation_benchmark.rs | 88 ++++++++++++++----- nexus/db-queries/src/db/datastore/sled.rs | 26 ++++++ nexus/db-queries/src/transaction_retry.rs | 2 +- nexus/tests/integration_tests/schema.rs | 4 +- 4 files changed, 96 insertions(+), 24 deletions(-) diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs index 3449fb68946..35a85f258f6 100644 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ b/nexus/db-queries/benches/sled_reservation_benchmark.rs @@ -166,6 +166,60 @@ struct TestHarness { db: TestDatabase, } +struct ContentionQuery { + sql: &'static str, + description: &'static str, +} + +const QUERIES: [ContentionQuery; 4] = [ + ContentionQuery { + sql: "SELECT table_name, index_name, num_contention_events::TEXT FROM crdb_internal.cluster_contended_indexes", + description: "Indexes which are experiencing contention", + }, + ContentionQuery { + sql: "SELECT table_name,num_contention_events::TEXT FROM crdb_internal.cluster_contended_tables", + description: "Tables which are experiencing contention", + }, + ContentionQuery { + sql: "WITH c AS (SELECT DISTINCT ON (table_id, index_id) table_id, index_id, num_contention_events AS events, cumulative_contention_time AS time FROM crdb_internal.cluster_contention_events) SELECT i.descriptor_name as table_name, i.index_name, c.events::TEXT, c.time::TEXT FROM crdb_internal.table_indexes AS i JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id ORDER BY c.time DESC LIMIT 10;", + description: "Top ten longest contention events, grouped by table + index", + }, + ContentionQuery { + // See: https://www.cockroachlabs.com/docs/v22.1/crdb-internal#example + // for the source here + sql: "SELECT DISTINCT + hce.blocking_statement, + substring(ss2.metadata ->> 'query', 1, 120) AS waiting_statement, + hce.contention_count::TEXT +FROM (SELECT + blocking_txn_fingerprint_id, + waiting_txn_fingerprint_id, + contention_count, + substring(ss.metadata ->> 'query', 1, 120) AS blocking_statement + FROM (SELECT + encode(blocking_txn_fingerprint_id, 'hex') as blocking_txn_fingerprint_id, + encode(waiting_txn_fingerprint_id, 'hex') as waiting_txn_fingerprint_id, + count(*) AS contention_count + FROM + crdb_internal.transaction_contention_events + GROUP BY + blocking_txn_fingerprint_id, waiting_txn_fingerprint_id + ), + crdb_internal.statement_statistics ss + WHERE + blocking_txn_fingerprint_id = encode(ss.transaction_fingerprint_id, 'hex')) hce, + crdb_internal.statement_statistics ss2 +WHERE + hce.blocking_txn_fingerprint_id != '0000000000000000' AND + hce.waiting_txn_fingerprint_id != '0000000000000000' AND + hce.waiting_txn_fingerprint_id = encode(ss2.transaction_fingerprint_id, 'hex') +ORDER BY + contention_count +DESC;", + description: "Transaction statements which are blocking other statements", + } +]; + impl TestHarness { async fn new(log: &Logger, sled_count: usize) -> Self { let db = TestDatabase::new_with_datastore(log).await; @@ -179,13 +233,8 @@ impl TestHarness { // Emit internal CockroachDb information about contention async fn print_contention(&self) { - let client = self.db.crdb().connect().await.expect("Failed to connect to db"); - - let queries = [ - "SELECT table_name, index_name, num_contention_events::TEXT FROM crdb_internal.cluster_contended_indexes", - "SELECT table_name,num_contention_events::TEXT FROM crdb_internal.cluster_contended_tables", - "WITH c AS (SELECT DISTINCT ON (table_id, index_id) table_id, index_id, num_contention_events AS events, cumulative_contention_time AS time FROM crdb_internal.cluster_contention_events) SELECT i.descriptor_name, i.index_name, c.events::TEXT, c.time::TEXT FROM crdb_internal.table_indexes AS i JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id ORDER BY c.time DESC LIMIT 10;" - ]; + let client = + self.db.crdb().connect().await.expect("Failed to connect to db"); // Used for padding: get a map of "column name" -> "max value length". let max_lengths_by_column = |rows: &Vec| { @@ -196,7 +245,8 @@ impl TestHarness { let name_len = column.name().len(); let len = std::cmp::max(value_len, name_len); - lengths.entry(column.name().to_string()) + lengths + .entry(column.name().to_string()) .and_modify(|entry| { if len > *entry { *entry = len; @@ -208,15 +258,17 @@ impl TestHarness { lengths }; - for sql in queries { - let rows = client.query(sql, &[]) - .await.expect("Failed to query contended tables"); + for ContentionQuery { sql, description } in QUERIES { + let rows = client + .query(sql, &[]) + .await + .expect("Failed to query contended tables"); let rows = process_rows(&rows); if rows.is_empty() { continue; } - println!("{sql}"); + println!("{description}"); let max_lengths = max_lengths_by_column(&rows); let mut header = true; @@ -249,11 +301,7 @@ impl TestHarness { } fn opctx(&self) -> Arc { - Arc::new( - self.db - .opctx() - .child(std::collections::BTreeMap::new()) - ) + Arc::new(self.db.opctx().child(std::collections::BTreeMap::new())) } fn db(&self) -> Arc { @@ -278,10 +326,8 @@ struct TestParams { tasks: usize, } -// const VMM_PARAMS: [usize; 3] = [1, 8, 16]; -// const TASK_PARAMS: [usize; 3] = [1, 2, 3]; -const VMM_PARAMS: [usize; 1] = [4]; -const TASK_PARAMS: [usize; 1] = [4]; +const VMM_PARAMS: [usize; 3] = [1, 8, 16]; +const TASK_PARAMS: [usize; 3] = [1, 4, 8]; ///////////////////////////////////////////////////////////////// // diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 4201bf7bc51..fc66915b1c0 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -484,6 +484,32 @@ impl DataStore { return Ok(old_resource[0].clone()); } + // Lock all "sled_resource_vmm" rows that we can read from. + // + // This operation is done only for performance reasons: + // - Creating a sled VMM resource reads all existing sled + // VMM resources, to determine space usage. + // - The operation ultimately concludes by INSERT-ing a new + // sled VMM resource row. + // - However, if there are concurrent transactions, this + // final INSERT invalidates the reads performed by other + // ongoing transactions, forcing them to restart. + // + // By locking these rows, we minimize contention, even + // though we're a potentially large number of rows. + let _ = resource_dsl::sled_resource_vmm + .inner_join( + sled_dsl::sled + .on(sled_dsl::id.eq(resource_dsl::sled_id)) + ) + .filter(sled_dsl::time_deleted.is_null()) + .sled_filter(SledFilter::ReservationCreate) + .select(resource_dsl::id) + .for_update() + .get_results_async::(&conn) + .await?; + + // If it doesn't already exist, find a sled with enough space // for the resources we're requesting. use db::schema::sled::dsl as sled_dsl; diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs index f626fcc80cd..cf8ee223767 100644 --- a/nexus/db-queries/src/transaction_retry.rs +++ b/nexus/db-queries/src/transaction_retry.rs @@ -75,7 +75,7 @@ pub struct RetryHelper { const MIN_RETRY_BACKOFF: Duration = Duration::from_millis(0); const MAX_RETRY_BACKOFF: Duration = Duration::from_millis(50); -const MAX_RETRY_ATTEMPTS: u32 = 50; +const MAX_RETRY_ATTEMPTS: u32 = 10; impl RetryHelper { /// Creates a new RetryHelper, and starts a timer tracking the transaction diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 50520e0a0de..fc0b840002a 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -12,11 +12,11 @@ use nexus_db_model::SCHEMA_VERSION as LATEST_SCHEMA_VERSION; use nexus_db_model::{AllSchemaVersions, SchemaVersion}; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DISALLOW_FULL_TABLE_SCAN_SQL; -use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; +use nexus_test_utils::sql::process_rows; use nexus_test_utils::sql::ColumnValue; use nexus_test_utils::sql::Row; use nexus_test_utils::sql::SqlEnum; -use nexus_test_utils::sql::process_rows; +use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; From 6704be18f2416ae4ce9c30ae68a65ef45b9822c0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 11 Feb 2025 12:51:24 -0800 Subject: [PATCH 35/84] restructure benchmark --- nexus/db-queries/Cargo.toml | 2 +- nexus/db-queries/benches/harness/db_utils.rs | 156 +++++ nexus/db-queries/benches/harness/mod.rs | 175 ++++++ nexus/db-queries/benches/sled_reservation.rs | 269 +++++++++ .../benches/sled_reservation_benchmark.rs | 561 ------------------ nexus/db-queries/src/db/datastore/sled.rs | 26 - 6 files changed, 601 insertions(+), 588 deletions(-) create mode 100644 nexus/db-queries/benches/harness/db_utils.rs create mode 100644 nexus/db-queries/benches/harness/mod.rs create mode 100644 nexus/db-queries/benches/sled_reservation.rs delete mode 100644 nexus/db-queries/benches/sled_reservation_benchmark.rs diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index a63cfcc223a..1c907019b68 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -101,5 +101,5 @@ subprocess.workspace = true term.workspace = true [[bench]] -name = "sled_reservation_benchmark" +name = "sled_reservation" harness = false diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs new file mode 100644 index 00000000000..e5dc85a4885 --- /dev/null +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -0,0 +1,156 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database test helpers +//! +//! These are largely ripped out of "nexus/db-queries/src/db/datastore". +//! +//! Benchmarks are compiled as external binaries from library crates, so we +//! can only access `pub` code. +//! +//! It may be worth refactoring some of these functions to a test utility +//! crate to avoid the de-duplication. + +use anyhow::Context; +use anyhow::Result; +use nexus_db_model::ByteCount; +use nexus_db_model::Generation; +use nexus_db_model::Project; +use nexus_db_model::Resources; +use nexus_db_model::Sled; +use nexus_db_model::SledBaseboard; +use nexus_db_model::SledReservationConstraintBuilder; +use nexus_db_model::SledSystemHardware; +use nexus_db_model::SledUpdate; +use nexus_db_queries::authz; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_types::external_api::params; +use omicron_common::api::external; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use std::net::Ipv6Addr; +use std::net::SocketAddrV6; +use uuid::Uuid; + +pub async fn create_project( + opctx: &OpContext, + datastore: &DataStore, +) -> (authz::Project, Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + + // Create a project + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: external::IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: "desc".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() +} + +pub fn rack_id() -> Uuid { + Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap() +} + +// Creates a "fake" Sled Baseboard. +pub fn sled_baseboard_for_test() -> SledBaseboard { + SledBaseboard { + serial_number: Uuid::new_v4().to_string(), + part_number: String::from("test-part"), + revision: 1, + } +} + +// Creates "fake" sled hardware accounting +pub fn sled_system_hardware_for_test() -> SledSystemHardware { + SledSystemHardware { + is_scrimlet: false, + usable_hardware_threads: 32, + usable_physical_ram: ByteCount::try_from(1 << 40).unwrap(), + reservoir_size: ByteCount::try_from(1 << 39).unwrap(), + } +} + +pub fn test_new_sled_update() -> SledUpdate { + let sled_id = Uuid::new_v4(); + let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); + let repo_depot_port = 0; + SledUpdate::new( + sled_id, + addr, + repo_depot_port, + sled_baseboard_for_test(), + sled_system_hardware_for_test(), + rack_id(), + Generation::new(), + ) +} + +pub async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { + let mut sleds = vec![]; + for _ in 0..count { + let (sled, _) = + datastore.sled_upsert(test_new_sled_update()).await.unwrap(); + sleds.push(sled); + } + sleds +} + +fn small_resource_request() -> Resources { + Resources::new( + 1, + // Just require the bare non-zero amount of RAM. + ByteCount::try_from(1024).unwrap(), + ByteCount::try_from(1024).unwrap(), + ) +} + +pub async fn create_reservation( + opctx: &OpContext, + db: &DataStore, +) -> Result { + let instance_id = InstanceUuid::new_v4(); + let vmm_id = PropolisUuid::new_v4(); + + loop { + match db.sled_reservation_create( + &opctx, + instance_id, + vmm_id, + small_resource_request(), + SledReservationConstraintBuilder::new().build(), + ) + .await { + Ok(_) => break, + Err(err) => { + // This condition is bad - it would result in a user-visible + // error, in most cases - but it's also an indication of failure + // due to contention. We normally bubble this out to users, + // rather than stalling the request, but in this particular + // case, we choose to retry immediately. + if err.to_string().contains("restart transaction") { + eprintln!("Warning: Transaction aborted due to contention"); + continue; + } + return Err(err).context("Failed to create reservation"); + } + } + }; + Ok(vmm_id) +} + +pub async fn delete_reservation( + opctx: &OpContext, + db: &DataStore, + vmm_id: PropolisUuid, +) -> Result<()> { + db.sled_reservation_delete(&opctx, vmm_id) + .await + .context("Failed to delete reservation")?; + Ok(()) +} diff --git a/nexus/db-queries/benches/harness/mod.rs b/nexus/db-queries/benches/harness/mod.rs new file mode 100644 index 00000000000..5219d418479 --- /dev/null +++ b/nexus/db-queries/benches/harness/mod.rs @@ -0,0 +1,175 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Shared Test Harness for benchmarking database queries +//! +//! This structure shares logic between benchmarks, making it easy +//! to perform shared tasks such as creating contention for reservations. + +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::pub_test_utils::TestDatabase; +use nexus_db_queries::db::DataStore; +use nexus_test_utils::sql::process_rows; +use nexus_test_utils::sql::Row; +use slog::Logger; +use std::collections::HashMap; +use std::sync::Arc; + +pub(crate) mod db_utils; + +use db_utils::*; + +pub struct TestHarness { + db: TestDatabase, +} + +struct ContentionQuery { + sql: &'static str, + description: &'static str, +} + +const QUERIES: [ContentionQuery; 4] = [ + ContentionQuery { + sql: "SELECT table_name, index_name, num_contention_events::TEXT FROM crdb_internal.cluster_contended_indexes", + description: "Indexes which are experiencing contention", + }, + ContentionQuery { + sql: "SELECT table_name,num_contention_events::TEXT FROM crdb_internal.cluster_contended_tables", + description: "Tables which are experiencing contention", + }, + ContentionQuery { + sql: "WITH c AS (SELECT DISTINCT ON (table_id, index_id) table_id, index_id, num_contention_events AS events, cumulative_contention_time AS time FROM crdb_internal.cluster_contention_events) SELECT i.descriptor_name as table_name, i.index_name, c.events::TEXT, c.time::TEXT FROM crdb_internal.table_indexes AS i JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id ORDER BY c.time DESC LIMIT 10;", + description: "Top ten longest contention events, grouped by table + index", + }, + ContentionQuery { + // See: https://www.cockroachlabs.com/docs/v22.1/crdb-internal#example + // for the source here + sql: "SELECT DISTINCT + hce.blocking_statement, + substring(ss2.metadata ->> 'query', 1, 120) AS waiting_statement, + hce.contention_count::TEXT +FROM (SELECT + blocking_txn_fingerprint_id, + waiting_txn_fingerprint_id, + contention_count, + substring(ss.metadata ->> 'query', 1, 120) AS blocking_statement + FROM (SELECT + encode(blocking_txn_fingerprint_id, 'hex') as blocking_txn_fingerprint_id, + encode(waiting_txn_fingerprint_id, 'hex') as waiting_txn_fingerprint_id, + count(*) AS contention_count + FROM + crdb_internal.transaction_contention_events + GROUP BY + blocking_txn_fingerprint_id, waiting_txn_fingerprint_id + ), + crdb_internal.statement_statistics ss + WHERE + blocking_txn_fingerprint_id = encode(ss.transaction_fingerprint_id, 'hex')) hce, + crdb_internal.statement_statistics ss2 +WHERE + hce.blocking_txn_fingerprint_id != '0000000000000000' AND + hce.waiting_txn_fingerprint_id != '0000000000000000' AND + hce.waiting_txn_fingerprint_id = encode(ss2.transaction_fingerprint_id, 'hex') +ORDER BY + contention_count +DESC;", + description: "Transaction statements which are blocking other statements", + } +]; + +impl TestHarness { + pub async fn new(log: &Logger, sled_count: usize) -> Self { + let db = TestDatabase::new_with_datastore(log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (_authz_project, _project) = + create_project(&opctx, &datastore).await; + create_sleds(&datastore, sled_count).await; + + Self { db } + } + + /// Emit internal CockroachDb information about contention + pub async fn print_contention(&self) { + let client = + self.db.crdb().connect().await.expect("Failed to connect to db"); + + // Used for padding: get a map of "column name" -> "max value length". + let max_lengths_by_column = |rows: &Vec| { + let mut lengths = HashMap::new(); + for row in rows { + for column in &row.values { + let value_len = column.value().unwrap().as_str().len(); + let name_len = column.name().len(); + let len = std::cmp::max(value_len, name_len); + + lengths + .entry(column.name().to_string()) + .and_modify(|entry| { + if len > *entry { + *entry = len; + } + }) + .or_insert(len); + } + } + lengths + }; + + for ContentionQuery { sql, description } in QUERIES { + let rows = client + .query(sql, &[]) + .await + .expect("Failed to query contended tables"); + let rows = process_rows(&rows); + if rows.is_empty() { + continue; + } + + println!("{description}"); + let max_lengths = max_lengths_by_column(&rows); + let mut header = true; + + for row in rows { + if header { + let mut total_len = 0; + for column in &row.values { + let width = max_lengths.get(column.name()).unwrap(); + print!(" {:width$} ", column.name()); + print!("|"); + total_len += width + 3; + } + println!(""); + println!("{:- Arc { + Arc::new(self.db.opctx().child(std::collections::BTreeMap::new())) + } + + /// Returns an owned reference to the datastore + pub fn db(&self) -> Arc { + self.db.datastore().clone() + } + + pub async fn terminate(self) { + self.db.terminate().await; + } +} + diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs new file mode 100644 index 00000000000..1c633aedc8a --- /dev/null +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -0,0 +1,269 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Benchmarks creating sled reservations + +use criterion::black_box; +use criterion::{criterion_group, criterion_main, Criterion}; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use omicron_test_utils::dev; +use slog::Logger; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; +use tokio::sync::Barrier; + +mod harness; + +use harness::db_utils::create_reservation; +use harness::db_utils::delete_reservation; +use harness::TestHarness; + +///////////////////////////////////////////////////////////////// +// +// PARAMETERS +// +// Describes varations between otherwise shared test logic + +#[derive(Copy, Clone)] +struct TestParams { + // Number of vmms to provision from the task-under-test + vmms: usize, + tasks: usize, +} + +const VMM_PARAMS: [usize; 3] = [1, 8, 16]; +const TASK_PARAMS: [usize; 3] = [1, 4, 8]; + +///////////////////////////////////////////////////////////////// +// +// BENCHMARKS +// +// You can run these with the following command: +// +// ```bash +// cargo bench -p nexus-db-queries +// ``` + +// Average a duration over a divisor. +// +// For example, if we reserve 100 vmms, you can use "100" as the divisor +// to get the "average duration to provision a vmm". +fn average_duration(duration: Duration, divisor: usize) -> Duration { + assert_ne!(divisor, 0, "Don't divide by zero please"); + + Duration::from_nanos( + u64::try_from(duration.as_nanos() / divisor as u128) + .expect("This benchmark is taking hundreds of years to run, maybe optimize it") + ) +} + +// Reserves "params.vmms" vmms, and later deletes their reservations. +// +// Returns the average time to provision a single vmm. +async fn reserve_vmms_and_return_average_duration( + params: &TestParams, + opctx: &OpContext, + db: &DataStore, +) -> Duration { + let mut vmm_ids = Vec::with_capacity(params.vmms); + let start = Instant::now(); + + // Clippy: We don't want to move this block outside of "black_box", even though it + // isn't returning anything. That would defeat the whole point of using "black_box", + // which is to avoid profiling code that is optimized based on the surrounding + // benchmark function. + #[allow(clippy::unit_arg)] + black_box({ + // Create all the requested vmms. + // + // Note that all prior reservations will remain in the DB as we continue + // provisioning the "next" vmm. + for _ in 0..params.vmms { + vmm_ids.push( + create_reservation(opctx, db) + .await + .expect("Failed to provision vmm"), + ); + } + }); + + // Return the "average time to provision a single vmm". + // + // This normalizes the results, regardless of how many vmms we are provisioning. + // + // Note that we expect additional contention to create more work, but it's difficult to + // normalize "how much work is being created by contention". + let duration = average_duration(start.elapsed(), params.vmms); + + // Clean up all our vmms. + // + // We don't really care how long this takes, so we omit it from the tracking time. + for vmm_id in vmm_ids.drain(..) { + delete_reservation(opctx, db, vmm_id) + .await + .expect("Failed to delete vmm"); + } + + duration +} + +async fn bench_reservation( + opctx: Arc, + db: Arc, + params: TestParams, + iterations: u64, +) -> Duration { + let duration = { + let mut total_duration = Duration::ZERO; + + // Each iteration is an "attempt" at the test. + for _ in 0..iterations { + // Within each attempt, we spawn the tasks requested. + let mut set = tokio::task::JoinSet::new(); + + // This barrier exists to lessen the impact of "task spawning" on the benchmark. + // + // We want to have all tasks run as concurrently as possible, since we're trying to + // measure contention explicitly. + let barrier = Arc::new(Barrier::new(params.tasks)); + + for _ in 0..params.tasks { + set.spawn({ + let opctx = opctx.clone(); + let db = db.clone(); + let barrier = barrier.clone(); + + async move { + // Wait until all tasks are ready... + barrier.wait().await; + + // ... and then actually do the benchmark + reserve_vmms_and_return_average_duration( + ¶ms, &opctx, &db, + ) + .await + } + }); + } + + // The sum of "average time to provision a single vmm" across all tasks. + let all_tasks_duration = set + .join_all() + .await + .into_iter() + .fold(Duration::ZERO, |acc, x| acc + x); + + // The normalized "time to provision a single vmm", across both: + // - The number of vmms reserved by each task, and + // - The number of tasks + // + // As an example, if we provision 10 vmms, and have 5 tasks, and we assume + // that VM provisioning time is exactly one second (contention has no impact, somehow): + // + // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average + // duration of "1 second". + // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds + // (1 second average * 5 tasks). + // - So, we'd increment our "total_duration" by "1 second per vmm", which has been + // normalized cross both the tasks and quantities of vmms. + // + // Why bother doing this? + // + // When we perform this normalization, we can vary the "total vmms provisioned" as well + // as "total tasks" significantly, but easily compare test durations with one another. + // + // For example: if the total number of vmms has no impact on the next provisioning + // request, we should see similar durations for "100 vmms reserved" vs "1 vmm + // reserved". However, if more vmms actually make reservation times slower, we'll see + // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: + total_duration += + average_duration(all_tasks_duration, params.tasks); + } + total_duration + }; + + duration +} + +// Typically we run our database tests using "cargo nextest run", +// which triggers the "crdb-seed" binary to create an initialized +// database when we boot up. +// +// If we're using "cargo bench", we don't get that guarantee. +// Go through the database ensuring process manually. +async fn setup_db(log: &Logger) { + print!("setting up seed cockroachdb database... "); + let (seed_tar, status) = dev::seed::ensure_seed_tarball_exists( + log, + dev::seed::should_invalidate_seed(), + ) + .await + .expect("Failed to create seed tarball for CRDB"); + status.log(log, &seed_tar); + unsafe { + std::env::set_var(dev::CRDB_SEED_TAR_ENV, seed_tar); + } + println!("OK"); +} + +fn sled_reservation_benchmark(c: &mut Criterion) { + let logctx = dev::test_setup_log("sled-reservation"); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(setup_db(&logctx.log)); + + let mut group = c.benchmark_group("vmm-reservation"); + for vmms in VMM_PARAMS { + for tasks in TASK_PARAMS { + let params = TestParams { vmms, tasks }; + let name = format!("{vmms}-vmms-{tasks}-tasks"); + + // Initialize the harness before calling "bench_function" so + // that the "warm-up" calls to "bench_function" are actually useful + // at warming up the database. + // + // This mitigates any database-caching issues like "loading schema + // on boot", or "connection pooling", as the pool stays the same + // between calls to the benchmark function. + let log = logctx.log.clone(); + let harness = rt.block_on(async move { + const SLED_COUNT: usize = 4; + TestHarness::new(&log, SLED_COUNT).await + }); + + // Actually invoke the benchmark. + group.bench_function(&name, |b| { + b.to_async(&rt).iter_custom(|iters| { + let opctx = harness.opctx(); + let db = harness.db(); + async move { bench_reservation(opctx, db, params, iters).await } + }) + }); + + // Clean-up the harness; we'll use a new database between + // varations in parameters. + rt.block_on(async move { + harness.print_contention().await; + harness.terminate().await; + }); + } + } + group.finish(); + logctx.cleanup_successful(); +} + +criterion_group!( + name = benches; + // To accomodate the fact that these benchmarks are a bit bulky, + // we set the following: + // - Smaller sample size, to keep running time down + // - Higher noise threshold, to avoid avoid false positive change detection + config = Criterion::default() + .sample_size(10) + .noise_threshold(0.10); + targets = sled_reservation_benchmark +); +criterion_main!(benches); diff --git a/nexus/db-queries/benches/sled_reservation_benchmark.rs b/nexus/db-queries/benches/sled_reservation_benchmark.rs deleted file mode 100644 index 35a85f258f6..00000000000 --- a/nexus/db-queries/benches/sled_reservation_benchmark.rs +++ /dev/null @@ -1,561 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Benchmarks creating sled reservations - -use anyhow::Context; -use anyhow::Result; -use criterion::black_box; -use criterion::{criterion_group, criterion_main, Criterion}; -use nexus_db_model::ByteCount; -use nexus_db_model::Generation; -use nexus_db_model::Project; -use nexus_db_model::Resources; -use nexus_db_model::Sled; -use nexus_db_model::SledBaseboard; -use nexus_db_model::SledReservationConstraintBuilder; -use nexus_db_model::SledSystemHardware; -use nexus_db_model::SledUpdate; -use nexus_db_queries::authz; -use nexus_db_queries::context::OpContext; -use nexus_db_queries::db::pub_test_utils::TestDatabase; -use nexus_db_queries::db::DataStore; -use nexus_test_utils::sql::process_rows; -use nexus_test_utils::sql::Row; -use nexus_types::external_api::params; -use omicron_common::api::external; -use omicron_test_utils::dev; -use omicron_uuid_kinds::InstanceUuid; -use omicron_uuid_kinds::PropolisUuid; -use slog::Logger; -use std::collections::HashMap; -use std::net::Ipv6Addr; -use std::net::SocketAddrV6; -use std::sync::Arc; -use std::time::Duration; -use std::time::Instant; -use tokio::sync::Barrier; -use uuid::Uuid; - -///////////////////////////////////////////////////////////////// -// -// TEST HELPERS -// -// These are largely ripped out of "nexus/db-queries/src/db/datastore". -// -// Benchmarks are compiled as external binaries from library crates, so we -// can only access `pub` code. -// -// It may be worth refactoring some of these functions to a test utility -// crate to avoid the de-duplication. - -async fn create_project( - opctx: &OpContext, - datastore: &DataStore, -) -> (authz::Project, Project) { - let authz_silo = opctx.authn.silo_required().unwrap(); - - // Create a project - let project = Project::new( - authz_silo.id(), - params::ProjectCreate { - identity: external::IdentityMetadataCreateParams { - name: "project".parse().unwrap(), - description: "desc".to_string(), - }, - }, - ); - datastore.project_create(&opctx, project).await.unwrap() -} - -fn rack_id() -> Uuid { - Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap() -} - -// Creates a "fake" Sled Baseboard. -fn sled_baseboard_for_test() -> SledBaseboard { - SledBaseboard { - serial_number: Uuid::new_v4().to_string(), - part_number: String::from("test-part"), - revision: 1, - } -} - -// Creates "fake" sled hardware accounting -fn sled_system_hardware_for_test() -> SledSystemHardware { - SledSystemHardware { - is_scrimlet: false, - usable_hardware_threads: 32, - usable_physical_ram: ByteCount::try_from(1 << 40).unwrap(), - reservoir_size: ByteCount::try_from(1 << 39).unwrap(), - } -} - -fn test_new_sled_update() -> SledUpdate { - let sled_id = Uuid::new_v4(); - let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); - let repo_depot_port = 0; - SledUpdate::new( - sled_id, - addr, - repo_depot_port, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id(), - Generation::new(), - ) -} - -async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { - let mut sleds = vec![]; - for _ in 0..count { - let (sled, _) = - datastore.sled_upsert(test_new_sled_update()).await.unwrap(); - sleds.push(sled); - } - sleds -} - -fn small_resource_request() -> Resources { - Resources::new( - 1, - // Just require the bare non-zero amount of RAM. - ByteCount::try_from(1024).unwrap(), - ByteCount::try_from(1024).unwrap(), - ) -} - -async fn create_reservation( - opctx: &OpContext, - db: &DataStore, -) -> Result { - let instance_id = InstanceUuid::new_v4(); - let vmm_id = PropolisUuid::new_v4(); - db.sled_reservation_create( - &opctx, - instance_id, - vmm_id, - small_resource_request(), - SledReservationConstraintBuilder::new().build(), - ) - .await - .context("Failed to create reservation")?; - Ok(vmm_id) -} - -async fn delete_reservation( - opctx: &OpContext, - db: &DataStore, - vmm_id: PropolisUuid, -) -> Result<()> { - db.sled_reservation_delete(&opctx, vmm_id) - .await - .context("Failed to delete reservation")?; - Ok(()) -} - -///////////////////////////////////////////////////////////////// -// -// TEST HARNESS -// -// This structure shares logic between benchmarks, making it easy -// to perform shared tasks such as creating contention for reservations. - -struct TestHarness { - db: TestDatabase, -} - -struct ContentionQuery { - sql: &'static str, - description: &'static str, -} - -const QUERIES: [ContentionQuery; 4] = [ - ContentionQuery { - sql: "SELECT table_name, index_name, num_contention_events::TEXT FROM crdb_internal.cluster_contended_indexes", - description: "Indexes which are experiencing contention", - }, - ContentionQuery { - sql: "SELECT table_name,num_contention_events::TEXT FROM crdb_internal.cluster_contended_tables", - description: "Tables which are experiencing contention", - }, - ContentionQuery { - sql: "WITH c AS (SELECT DISTINCT ON (table_id, index_id) table_id, index_id, num_contention_events AS events, cumulative_contention_time AS time FROM crdb_internal.cluster_contention_events) SELECT i.descriptor_name as table_name, i.index_name, c.events::TEXT, c.time::TEXT FROM crdb_internal.table_indexes AS i JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id ORDER BY c.time DESC LIMIT 10;", - description: "Top ten longest contention events, grouped by table + index", - }, - ContentionQuery { - // See: https://www.cockroachlabs.com/docs/v22.1/crdb-internal#example - // for the source here - sql: "SELECT DISTINCT - hce.blocking_statement, - substring(ss2.metadata ->> 'query', 1, 120) AS waiting_statement, - hce.contention_count::TEXT -FROM (SELECT - blocking_txn_fingerprint_id, - waiting_txn_fingerprint_id, - contention_count, - substring(ss.metadata ->> 'query', 1, 120) AS blocking_statement - FROM (SELECT - encode(blocking_txn_fingerprint_id, 'hex') as blocking_txn_fingerprint_id, - encode(waiting_txn_fingerprint_id, 'hex') as waiting_txn_fingerprint_id, - count(*) AS contention_count - FROM - crdb_internal.transaction_contention_events - GROUP BY - blocking_txn_fingerprint_id, waiting_txn_fingerprint_id - ), - crdb_internal.statement_statistics ss - WHERE - blocking_txn_fingerprint_id = encode(ss.transaction_fingerprint_id, 'hex')) hce, - crdb_internal.statement_statistics ss2 -WHERE - hce.blocking_txn_fingerprint_id != '0000000000000000' AND - hce.waiting_txn_fingerprint_id != '0000000000000000' AND - hce.waiting_txn_fingerprint_id = encode(ss2.transaction_fingerprint_id, 'hex') -ORDER BY - contention_count -DESC;", - description: "Transaction statements which are blocking other statements", - } -]; - -impl TestHarness { - async fn new(log: &Logger, sled_count: usize) -> Self { - let db = TestDatabase::new_with_datastore(log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - let (_authz_project, _project) = - create_project(&opctx, &datastore).await; - create_sleds(&datastore, sled_count).await; - - Self { db } - } - - // Emit internal CockroachDb information about contention - async fn print_contention(&self) { - let client = - self.db.crdb().connect().await.expect("Failed to connect to db"); - - // Used for padding: get a map of "column name" -> "max value length". - let max_lengths_by_column = |rows: &Vec| { - let mut lengths = HashMap::new(); - for row in rows { - for column in &row.values { - let value_len = column.value().unwrap().as_str().len(); - let name_len = column.name().len(); - let len = std::cmp::max(value_len, name_len); - - lengths - .entry(column.name().to_string()) - .and_modify(|entry| { - if len > *entry { - *entry = len; - } - }) - .or_insert(len); - } - } - lengths - }; - - for ContentionQuery { sql, description } in QUERIES { - let rows = client - .query(sql, &[]) - .await - .expect("Failed to query contended tables"); - let rows = process_rows(&rows); - if rows.is_empty() { - continue; - } - - println!("{description}"); - let max_lengths = max_lengths_by_column(&rows); - let mut header = true; - - for row in rows { - if header { - let mut total_len = 0; - for column in &row.values { - let width = max_lengths.get(column.name()).unwrap(); - print!(" {:width$} ", column.name()); - print!("|"); - total_len += width + 3; - } - println!(""); - println!("{:- Arc { - Arc::new(self.db.opctx().child(std::collections::BTreeMap::new())) - } - - fn db(&self) -> Arc { - self.db.datastore().clone() - } - - async fn terminate(self) { - self.db.terminate().await; - } -} - -///////////////////////////////////////////////////////////////// -// -// PARAMETERS -// -// Describes varations between otherwise shared test logic - -#[derive(Copy, Clone)] -struct TestParams { - // Number of vmms to provision from the task-under-test - vmms: usize, - tasks: usize, -} - -const VMM_PARAMS: [usize; 3] = [1, 8, 16]; -const TASK_PARAMS: [usize; 3] = [1, 4, 8]; - -///////////////////////////////////////////////////////////////// -// -// BENCHMARKS -// -// You can run these with the following command: -// -// ```bash -// cargo bench -p nexus-db-queries -// ``` - -// Average a duration over a divisor. -// -// For example, if we reserve 100 vmms, you can use "100" as the divisor -// to get the "average duration to provision a vmm". -fn average_duration(duration: Duration, divisor: usize) -> Duration { - assert_ne!(divisor, 0, "Don't divide by zero please"); - - Duration::from_nanos( - u64::try_from(duration.as_nanos() / divisor as u128) - .expect("This benchmark is taking hundreds of years to run, maybe optimize it") - ) -} - -// Reserves "params.vmms" vmms, and later deletes their reservations. -// -// Returns the average time to provision a single vmm. -async fn reserve_vmms_and_return_average_duration( - params: &TestParams, - opctx: &OpContext, - db: &DataStore, -) -> Duration { - let mut vmm_ids = Vec::with_capacity(params.vmms); - let start = Instant::now(); - - // Clippy: We don't want to move this block outside of "black_box", even though it - // isn't returning anything. That would defeat the whole point of using "black_box", - // which is to avoid profiling code that is optimized based on the surrounding - // benchmark function. - #[allow(clippy::unit_arg)] - black_box({ - // Create all the requested vmms. - // - // Note that all prior reservations will remain in the DB as we continue - // provisioning the "next" vmm. - for _ in 0..params.vmms { - vmm_ids.push( - create_reservation(opctx, db) - .await - .expect("Failed to provision vmm"), - ); - } - }); - - // Return the "average time to provision a single vmm". - // - // This normalizes the results, regardless of how many vmms we are provisioning. - // - // Note that we expect additional contention to create more work, but it's difficult to - // normalize "how much work is being created by contention". - let duration = average_duration(start.elapsed(), params.vmms); - - // Clean up all our vmms. - // - // We don't really care how long this takes, so we omit it from the tracking time. - for vmm_id in vmm_ids.drain(..) { - delete_reservation(opctx, db, vmm_id) - .await - .expect("Failed to delete vmm"); - } - - duration -} - -async fn bench_reservation( - opctx: Arc, - db: Arc, - params: TestParams, - iterations: u64, -) -> Duration { - let duration = { - let mut total_duration = Duration::ZERO; - - // Each iteration is an "attempt" at the test. - for _ in 0..iterations { - // Within each attempt, we spawn the tasks requested. - let mut set = tokio::task::JoinSet::new(); - - // This barrier exists to lessen the impact of "task spawning" on the benchmark. - // - // We want to have all tasks run as concurrently as possible, since we're trying to - // measure contention explicitly. - let barrier = Arc::new(Barrier::new(params.tasks)); - - for _ in 0..params.tasks { - set.spawn({ - let opctx = opctx.clone(); - let db = db.clone(); - let barrier = barrier.clone(); - - async move { - // Wait until all tasks are ready... - barrier.wait().await; - - // ... and then actually do the benchmark - reserve_vmms_and_return_average_duration( - ¶ms, &opctx, &db, - ) - .await - } - }); - } - - // The sum of "average time to provision a single vmm" across all tasks. - let all_tasks_duration = set - .join_all() - .await - .into_iter() - .fold(Duration::ZERO, |acc, x| acc + x); - - // The normalized "time to provision a single vmm", across both: - // - The number of vmms reserved by each task, and - // - The number of tasks - // - // As an example, if we provision 10 vmms, and have 5 tasks, and we assume - // that VM provisioning time is exactly one second (contention has no impact, somehow): - // - // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average - // duration of "1 second". - // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds - // (1 second average * 5 tasks). - // - So, we'd increment our "total_duration" by "1 second per vmm", which has been - // normalized cross both the tasks and quantities of vmms. - // - // Why bother doing this? - // - // When we perform this normalization, we can vary the "total vmms provisioned" as well - // as "total tasks" significantly, but easily compare test durations with one another. - // - // For example: if the total number of vmms has no impact on the next provisioning - // request, we should see similar durations for "100 vmms reserved" vs "1 vmm - // reserved". However, if more vmms actually make reservation times slower, we'll see - // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: - total_duration += - average_duration(all_tasks_duration, params.tasks); - } - total_duration - }; - - duration -} - -// Typically we run our database tests using "cargo nextest run", -// which triggers the "crdb-seed" binary to create an initialized -// database when we boot up. -// -// If we're using "cargo bench", we don't get that guarantee. -// Go through the database ensuring process manually. -async fn setup_db(log: &Logger) { - print!("setting up seed cockroachdb database... "); - let (seed_tar, status) = dev::seed::ensure_seed_tarball_exists( - log, - dev::seed::should_invalidate_seed(), - ) - .await - .expect("Failed to create seed tarball for CRDB"); - status.log(log, &seed_tar); - unsafe { - std::env::set_var(dev::CRDB_SEED_TAR_ENV, seed_tar); - } - println!("OK"); -} - -fn sled_reservation_benchmark(c: &mut Criterion) { - let logctx = dev::test_setup_log("sled-reservation"); - - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(setup_db(&logctx.log)); - - let mut group = c.benchmark_group("vmm-reservation"); - for vmms in VMM_PARAMS { - for tasks in TASK_PARAMS { - let params = TestParams { vmms, tasks }; - let name = format!("{vmms}-vmms-{tasks}-tasks"); - - // Initialize the harness before calling "bench_function" so - // that the "warm-up" calls to "bench_function" are actually useful - // at warming up the database. - // - // This mitigates any database-caching issues like "loading schema - // on boot", or "connection pooling", as the pool stays the same - // between calls to the benchmark function. - let log = logctx.log.clone(); - let harness = rt.block_on(async move { - const SLED_COUNT: usize = 4; - TestHarness::new(&log, SLED_COUNT).await - }); - - // Actually invoke the benchmark. - group.bench_function(&name, |b| { - b.to_async(&rt).iter_custom(|iters| { - let opctx = harness.opctx(); - let db = harness.db(); - async move { bench_reservation(opctx, db, params, iters).await } - }) - }); - - // Clean-up the harness; we'll use a new database between - // varations in parameters. - rt.block_on(async move { - harness.print_contention().await; - harness.terminate().await; - }); - } - } - group.finish(); - logctx.cleanup_successful(); -} - -criterion_group!( - name = benches; - // To accomodate the fact that these benchmarks are a bit bulky, - // we set the following: - // - Smaller sample size, to keep running time down - // - Higher noise threshold, to avoid avoid false positive change detection - config = Criterion::default() - .sample_size(10) - .noise_threshold(0.10); - targets = sled_reservation_benchmark -); -criterion_main!(benches); diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index fc66915b1c0..4201bf7bc51 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -484,32 +484,6 @@ impl DataStore { return Ok(old_resource[0].clone()); } - // Lock all "sled_resource_vmm" rows that we can read from. - // - // This operation is done only for performance reasons: - // - Creating a sled VMM resource reads all existing sled - // VMM resources, to determine space usage. - // - The operation ultimately concludes by INSERT-ing a new - // sled VMM resource row. - // - However, if there are concurrent transactions, this - // final INSERT invalidates the reads performed by other - // ongoing transactions, forcing them to restart. - // - // By locking these rows, we minimize contention, even - // though we're a potentially large number of rows. - let _ = resource_dsl::sled_resource_vmm - .inner_join( - sled_dsl::sled - .on(sled_dsl::id.eq(resource_dsl::sled_id)) - ) - .filter(sled_dsl::time_deleted.is_null()) - .sled_filter(SledFilter::ReservationCreate) - .select(resource_dsl::id) - .for_update() - .get_results_async::(&conn) - .await?; - - // If it doesn't already exist, find a sled with enough space // for the resources we're requesting. use db::schema::sled::dsl as sled_dsl; From 127285c77e143e42b3cea91020b2d3077a60bd17 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 11 Feb 2025 12:56:57 -0800 Subject: [PATCH 36/84] more refactoring --- nexus/db-queries/benches/harness/db_utils.rs | 20 ++++++++------- nexus/db-queries/benches/harness/mod.rs | 21 ++++++++++++++++ nexus/db-queries/benches/sled_reservation.rs | 26 ++------------------ 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index e5dc85a4885..bf642ad9021 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -118,14 +118,16 @@ pub async fn create_reservation( let vmm_id = PropolisUuid::new_v4(); loop { - match db.sled_reservation_create( - &opctx, - instance_id, - vmm_id, - small_resource_request(), - SledReservationConstraintBuilder::new().build(), - ) - .await { + match db + .sled_reservation_create( + &opctx, + instance_id, + vmm_id, + small_resource_request(), + SledReservationConstraintBuilder::new().build(), + ) + .await + { Ok(_) => break, Err(err) => { // This condition is bad - it would result in a user-visible @@ -140,7 +142,7 @@ pub async fn create_reservation( return Err(err).context("Failed to create reservation"); } } - }; + } Ok(vmm_id) } diff --git a/nexus/db-queries/benches/harness/mod.rs b/nexus/db-queries/benches/harness/mod.rs index 5219d418479..1b244e083ae 100644 --- a/nexus/db-queries/benches/harness/mod.rs +++ b/nexus/db-queries/benches/harness/mod.rs @@ -12,6 +12,7 @@ use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DataStore; use nexus_test_utils::sql::process_rows; use nexus_test_utils::sql::Row; +use omicron_test_utils::dev; use slog::Logger; use std::collections::HashMap; use std::sync::Arc; @@ -173,3 +174,23 @@ impl TestHarness { } } +/// Typically we run our database tests using "cargo nextest run", +/// which triggers the "crdb-seed" binary to create an initialized +/// database when we boot up. +/// +/// If we're using "cargo bench", we don't get that guarantee. +/// Go through the database ensuring process manually. +pub async fn setup_db(log: &Logger) { + print!("setting up seed cockroachdb database... "); + let (seed_tar, status) = dev::seed::ensure_seed_tarball_exists( + log, + dev::seed::should_invalidate_seed(), + ) + .await + .expect("Failed to create seed tarball for CRDB"); + status.log(log, &seed_tar); + unsafe { + std::env::set_var(dev::CRDB_SEED_TAR_ENV, seed_tar); + } + println!("OK"); +} diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 1c633aedc8a..477f48e8271 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -9,7 +9,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use omicron_test_utils::dev; -use slog::Logger; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -56,7 +55,7 @@ fn average_duration(duration: Duration, divisor: usize) -> Duration { Duration::from_nanos( u64::try_from(duration.as_nanos() / divisor as u128) - .expect("This benchmark is taking hundreds of years to run, maybe optimize it") + .expect("This benchmark is taking hundreds of years to run, maybe optimize it?") ) } @@ -188,32 +187,11 @@ async fn bench_reservation( duration } -// Typically we run our database tests using "cargo nextest run", -// which triggers the "crdb-seed" binary to create an initialized -// database when we boot up. -// -// If we're using "cargo bench", we don't get that guarantee. -// Go through the database ensuring process manually. -async fn setup_db(log: &Logger) { - print!("setting up seed cockroachdb database... "); - let (seed_tar, status) = dev::seed::ensure_seed_tarball_exists( - log, - dev::seed::should_invalidate_seed(), - ) - .await - .expect("Failed to create seed tarball for CRDB"); - status.log(log, &seed_tar); - unsafe { - std::env::set_var(dev::CRDB_SEED_TAR_ENV, seed_tar); - } - println!("OK"); -} - fn sled_reservation_benchmark(c: &mut Criterion) { let logctx = dev::test_setup_log("sled-reservation"); let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(setup_db(&logctx.log)); + rt.block_on(harness::setup_db(&logctx.log)); let mut group = c.benchmark_group("vmm-reservation"); for vmms in VMM_PARAMS { From acf3a64f8aaca81fd9efb8bc0a4f797c73428206 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 11:19:04 -0800 Subject: [PATCH 37/84] test naming --- nexus/db-queries/benches/harness/db_utils.rs | 1 - nexus/db-queries/benches/sled_reservation.rs | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index bf642ad9021..a714d462b2d 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -136,7 +136,6 @@ pub async fn create_reservation( // rather than stalling the request, but in this particular // case, we choose to retry immediately. if err.to_string().contains("restart transaction") { - eprintln!("Warning: Transaction aborted due to contention"); continue; } return Err(err).context("Failed to create reservation"); diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 477f48e8271..4b1d825beab 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -194,10 +194,10 @@ fn sled_reservation_benchmark(c: &mut Criterion) { rt.block_on(harness::setup_db(&logctx.log)); let mut group = c.benchmark_group("vmm-reservation"); - for vmms in VMM_PARAMS { - for tasks in TASK_PARAMS { + for tasks in TASK_PARAMS { + for vmms in VMM_PARAMS { let params = TestParams { vmms, tasks }; - let name = format!("{vmms}-vmms-{tasks}-tasks"); + let name = format!("{tasks}-tasks-{vmms}-vmms"); // Initialize the harness before calling "bench_function" so // that the "warm-up" calls to "bench_function" are actually useful From 2eaef4fd922a52bbf19130d6fd7bf385265f5db6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 11:19:41 -0800 Subject: [PATCH 38/84] Start working towards a reduced-contention sled reservation --- nexus/db-queries/src/db/datastore/sled.rs | 156 +++++--------- nexus/db-queries/src/db/queries/mod.rs | 1 + .../src/db/queries/sled_reservation.rs | 202 ++++++++++++++++++ .../tests/output/sled_find_targets_query.sql | 90 ++++++++ 4 files changed, 344 insertions(+), 105 deletions(-) create mode 100644 nexus/db-queries/src/db/queries/sled_reservation.rs create mode 100644 nexus/db-queries/tests/output/sled_find_targets_query.sql diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 4201bf7bc51..0d3c82b6b12 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -21,8 +21,7 @@ use crate::db::model::SledUpdate; use crate::db::pagination::paginated; use crate::db::pagination::Paginator; use crate::db::pool::DbConnection; -use crate::db::queries::affinity::lookup_affinity_sleds_query; -use crate::db::queries::affinity::lookup_anti_affinity_sleds_query; +use crate::db::queries::sled_reservation::sled_find_targets_query; use crate::db::update_and_check::{UpdateAndCheck, UpdateStatus}; use crate::db::TransactionError; use crate::transaction_retry::OptionalError; @@ -208,24 +207,20 @@ enum SledReservationTransactionError { fn pick_sled_reservation_target( log: &Logger, mut targets: HashSet, - anti_affinity_sleds: Vec<(AffinityPolicy, Uuid)>, - affinity_sleds: Vec<(AffinityPolicy, Uuid)>, + anti_affinity_sleds: Vec<(AffinityPolicy, SledUuid)>, + affinity_sleds: Vec<(AffinityPolicy, SledUuid)>, ) -> Result { let (banned, mut unpreferred): (HashSet<_>, HashSet<_>) = anti_affinity_sleds.into_iter().partition_map(|(policy, id)| { - let id = SledUuid::from_untyped_uuid(id); match policy { AffinityPolicy::Fail => Either::Left(id), AffinityPolicy::Allow => Either::Right(id), } }); let (required, mut preferred): (HashSet<_>, HashSet<_>) = - affinity_sleds.into_iter().partition_map(|(policy, id)| { - let id = SledUuid::from_untyped_uuid(id); - match policy { - AffinityPolicy::Fail => Either::Left(id), - AffinityPolicy::Allow => Either::Right(id), - } + affinity_sleds.into_iter().partition_map(|(policy, id)| match policy { + AffinityPolicy::Fail => Either::Left(id), + AffinityPolicy::Allow => Either::Right(id), }); if !banned.is_empty() { @@ -463,15 +458,23 @@ impl DataStore { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; + let must_use_sleds: HashSet = constraints + .must_select_from() + .into_iter() + .flatten() + .map(|id| SledUuid::from_untyped_uuid(*id)) + .collect(); + self.transaction_retry_wrapper("sled_reservation_create") .transaction(&conn, |conn| { // Clone variables into retryable function let err = err.clone(); - let constraints = constraints.clone(); + let must_use_sleds = must_use_sleds.clone(); let resources = resources.clone(); async move { use db::schema::sled_resource_vmm::dsl as resource_dsl; + // Check if resource ID already exists - if so, return it. let old_resource = resource_dsl::sled_resource_vmm .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) @@ -484,103 +487,46 @@ impl DataStore { return Ok(old_resource[0].clone()); } - // If it doesn't already exist, find a sled with enough space - // for the resources we're requesting. - use db::schema::sled::dsl as sled_dsl; - // This answers the boolean question: - // "Does the SUM of all hardware thread usage, plus the one we're trying - // to allocate, consume less threads than exists on the sled?" - let sled_has_space_for_threads = - (diesel::dsl::sql::( - &format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::hardware_threads::NAME - ), - ) + resources.hardware_threads) - .le(sled_dsl::usable_hardware_threads); - - // This answers the boolean question: - // "Does the SUM of all RAM usage, plus the one we're trying - // to allocate, consume less RAM than exists on the sled?" - let sled_has_space_for_rss = - (diesel::dsl::sql::( - &format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::rss_ram::NAME - ), - ) + resources.rss_ram) - .le(sled_dsl::usable_physical_ram); - - // Determine whether adding this service's reservoir allocation - // to what's allocated on the sled would avoid going over quota. - let sled_has_space_in_reservoir = - (diesel::dsl::sql::( - &format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::reservoir_ram::NAME - ), - ) + resources.reservoir_ram) - .le(sled_dsl::reservoir_size); - - // Generate a query describing all of the sleds that have space - // for this reservation. - let mut sled_targets = sled_dsl::sled - .left_join( - resource_dsl::sled_resource_vmm - .on(resource_dsl::sled_id.eq(sled_dsl::id)), - ) - .group_by(sled_dsl::id) - .having( - sled_has_space_for_threads - .and(sled_has_space_for_rss) - .and(sled_has_space_in_reservoir), - ) - .filter(sled_dsl::time_deleted.is_null()) - // Ensure that reservations can be created on the sled. - .sled_filter(SledFilter::ReservationCreate) - .select(sled_dsl::id) - .into_boxed(); - - // Further constrain the sled IDs according to any caller- - // supplied constraints. - if let Some(must_select_from) = - constraints.must_select_from() - { - sled_targets = sled_targets.filter( - sled_dsl::id.eq_any(must_select_from.to_vec()), - ); + let possible_sleds = sled_find_targets_query(instance_id, &resources) + .get_results_async::<( + // Sled UUID + Uuid, + // Would an allocation to this sled fit? + bool, + // Affinity policy on this sled + Option, + // Anti-affinity policy on this sled + Option, + )>(&conn).await?; + + println!("Observed sleds: {possible_sleds:#?}"); + + let mut sled_targets = HashSet::new(); + let mut affinity_sleds = vec![]; + let mut anti_affinity_sleds = vec![]; + + for ( + sled_id, + fits, + affinity_policy, + anti_affinity_policy, + ) in possible_sleds { + let sled_id = SledUuid::from_untyped_uuid(sled_id); + + if fits && (must_use_sleds.is_empty() || must_use_sleds.contains(&sled_id)) { + sled_targets.insert(sled_id); + } + if let Some(policy) = affinity_policy { + affinity_sleds.push((policy, sled_id)); + } + if let Some(policy) = anti_affinity_policy { + anti_affinity_sleds.push((policy, sled_id)); + } } - define_sql_function!(fn random() -> diesel::sql_types::Float); - - // Fetch all viable sled targets - let sled_targets = sled_targets - .order(random()) - .get_results_async::(&conn) - .await?; - - info!( - opctx.log, - "found {} available sled targets before considering affinity", sled_targets.len(); - "sled_ids" => ?sled_targets, - ); - - let anti_affinity_sleds = lookup_anti_affinity_sleds_query( - instance_id, - ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; - - let affinity_sleds = lookup_affinity_sleds_query( - instance_id, - ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; - - let targets: HashSet = sled_targets - .into_iter() - .map(|id| SledUuid::from_untyped_uuid(id)) - .collect(); - let sled_target = pick_sled_reservation_target( &opctx.log, - targets, + sled_targets, anti_affinity_sleds, affinity_sleds, ).map_err(|e| { diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index f2af3aebc99..128c73896ad 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -14,6 +14,7 @@ mod next_item; pub mod network_interface; pub mod oximeter; pub mod region_allocation; +pub mod sled_reservation; pub mod virtual_provisioning_collection_update; pub mod vpc; pub mod vpc_subnet; diff --git a/nexus/db-queries/src/db/queries/sled_reservation.rs b/nexus/db-queries/src/db/queries/sled_reservation.rs new file mode 100644 index 00000000000..b5ba7ba880b --- /dev/null +++ b/nexus/db-queries/src/db/queries/sled_reservation.rs @@ -0,0 +1,202 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of queries for affinity groups + +use crate::db::model::AffinityPolicyEnum; +use crate::db::model::Resources; +use crate::db::raw_query_builder::QueryBuilder; +use crate::db::raw_query_builder::TypedSqlQuery; +use diesel::sql_types; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; + +/// Return all possible Sleds where we might perform allocation +/// +/// The rows returned by this CTE indicate: +/// +/// - The Sled which we're considering +/// - A bool indicating whether the allocation fits +/// - Affinity Policy +/// - Anti-Affinity Policy +/// +/// Generally, we'd like to only return sleds where an allocation of [Resources] +/// could fit. However, we also need to observe all affinity groups, in case +/// affinity group policy forces us to allocate on a sled where insufficient +/// space exists. +/// +/// Note that we don't bother checking if the VMM has already been provisioned, +/// we're just searching for spots where it might fit. +pub fn sled_find_targets_query( + instance_id: InstanceUuid, + resources: &Resources, +) -> TypedSqlQuery<( + sql_types::Uuid, + sql_types::Bool, + sql_types::Nullable, + sql_types::Nullable, +)> { + QueryBuilder::new().sql(" + WITH sled_targets AS ( + SELECT sled.id as sled_id + FROM sled + LEFT JOIN sled_resource_vmm + ON sled_resource_vmm.sled_id = sled.id + WHERE + sled.time_deleted IS NULL AND + sled.sled_policy = 'in_service' AND + sled.sled_state = 'active' + GROUP BY sled.id + HAVING + COALESCE(SUM(CAST(sled_resource_vmm.hardware_threads AS INT8)), 0) + " + ).param().sql(" <= sled.usable_hardware_threads AND + COALESCE(SUM(CAST(sled_resource_vmm.rss_ram AS INT8)), 0) + " + ).param().sql(" <= sled.usable_physical_ram AND + COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " + ).param().sql(" <= sled.reservoir_size + ), + our_aa_groups AS ( + SELECT group_id + FROM anti_affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_aa_instances AS ( + SELECT anti_affinity_group_instance_membership.group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + WHERE instance_id != ").param().sql(" + ), + other_aa_instances_by_policy AS ( + SELECT policy,instance_id + FROM other_aa_instances + JOIN anti_affinity_group + ON + anti_affinity_group.id = other_aa_instances.group_id AND + anti_affinity_group.failure_domain = 'sled' + WHERE anti_affinity_group.time_deleted IS NULL + ), + aa_policy_and_sleds AS ( + SELECT DISTINCT policy,sled_id + FROM other_aa_instances_by_policy + JOIN sled_resource_vmm + ON + sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id + ), + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), + other_a_instances_by_policy AS ( + SELECT policy,instance_id + FROM other_a_instances + JOIN affinity_group + ON + affinity_group.id = other_a_instances.group_id AND + affinity_group.failure_domain = 'sled' + WHERE affinity_group.time_deleted IS NULL + ), + a_policy_and_sleds AS ( + SELECT DISTINCT policy,sled_id + FROM other_a_instances_by_policy + JOIN sled_resource_vmm + ON + sled_resource_vmm.instance_id = other_a_instances_by_policy.instance_id + ), + sleds_with_space AS ( + SELECT + s.sled_id, + a.policy as a_policy, + aa.policy as aa_policy + FROM + sled_targets s + LEFT JOIN a_policy_and_sleds a ON a.sled_id = s.sled_id + LEFT JOIN aa_policy_and_sleds aa ON aa.sled_id = s.sled_id + ), + sleds_without_space AS ( + SELECT + sled_id, + policy as a_policy, + NULL as aa_policy + FROM + a_policy_and_sleds + WHERE + a_policy_and_sleds.sled_id NOT IN (SELECT sled_id from sleds_with_space) + ) + SELECT sled_id, TRUE, a_policy, aa_policy FROM sleds_with_space + UNION + SELECT sled_id, FALSE, a_policy, aa_policy FROM sleds_without_space + ") + .bind::(resources.hardware_threads) + .bind::(resources.rss_ram) + .bind::(resources.reservoir_ram) + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .query() +} + +#[cfg(test)] +mod test { + use super::*; + use crate::db::explain::ExplainableAsync; + use crate::db::model; + use crate::db::pub_test_utils::TestDatabase; + use crate::db::raw_query_builder::expectorate_query_contents; + + use omicron_common::api::external; + use omicron_test_utils::dev; + + #[tokio::test] + async fn expectorate_sled_find_targets_query() { + let id = InstanceUuid::nil(); + let resources = Resources::new( + 0, + model::ByteCount::from(external::ByteCount::from_gibibytes_u32(0)), + model::ByteCount::from(external::ByteCount::from_gibibytes_u32(0)), + ); + + let query = sled_find_targets_query(id, &resources); + expectorate_query_contents( + &query, + "tests/output/sled_find_targets_query.sql", + ) + .await; + } + + #[tokio::test] + async fn explain_sled_find_targets_query() { + let logctx = dev::test_setup_log("explain_sled_find_targets_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let id = InstanceUuid::nil(); + let resources = Resources::new( + 0, + model::ByteCount::from(external::ByteCount::from_gibibytes_u32(0)), + model::ByteCount::from(external::ByteCount::from_gibibytes_u32(0)), + ); + + let query = sled_find_targets_query(id, &resources); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // NOTE: There are some tests you might wanna nab from "affinity.rs" +} diff --git a/nexus/db-queries/tests/output/sled_find_targets_query.sql b/nexus/db-queries/tests/output/sled_find_targets_query.sql new file mode 100644 index 00000000000..ca9e26bbcba --- /dev/null +++ b/nexus/db-queries/tests/output/sled_find_targets_query.sql @@ -0,0 +1,90 @@ +WITH + sled_targets + AS ( + SELECT + sled.id AS sled_id + FROM + sled LEFT JOIN sled_resource_vmm ON sled_resource_vmm.sled_id = sled.id + WHERE + sled.time_deleted IS NULL AND sled.sled_policy = 'in_service' AND sled.sled_state = 'active' + GROUP BY + sled.id + HAVING + COALESCE(sum(CAST(sled_resource_vmm.hardware_threads AS INT8)), 0) + $1 + <= sled.usable_hardware_threads + AND COALESCE(sum(CAST(sled_resource_vmm.rss_ram AS INT8)), 0) + $2 + <= sled.usable_physical_ram + AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $3 + <= sled.reservoir_size + ), + our_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $4), + other_aa_instances + AS ( + SELECT + anti_affinity_group_instance_membership.group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + WHERE + instance_id != $5 + ), + other_aa_instances_by_policy + AS ( + SELECT + policy, instance_id + FROM + other_aa_instances + JOIN anti_affinity_group ON + anti_affinity_group.id = other_aa_instances.group_id + AND anti_affinity_group.failure_domain = 'sled' + WHERE + anti_affinity_group.time_deleted IS NULL + ), + aa_policy_and_sleds + AS ( + SELECT + DISTINCT policy, sled_id + FROM + other_aa_instances_by_policy + JOIN sled_resource_vmm ON + sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id + ), + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $6), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $7 + ), + other_a_instances_by_policy + AS ( + SELECT + policy, instance_id + FROM + other_a_instances + JOIN affinity_group ON + affinity_group.id = other_a_instances.group_id + AND affinity_group.failure_domain = 'sled' + WHERE + affinity_group.time_deleted IS NULL + ), + a_policy_and_sleds + AS ( + SELECT + DISTINCT policy, sled_id + FROM + other_a_instances_by_policy + JOIN sled_resource_vmm ON + sled_resource_vmm.instance_id = other_a_instances_by_policy.instance_id + ) +SELECT + a_policy_and_sleds.policy, aa_policy_and_sleds.policy, sled_targets.sled_id +FROM + (sled_targets LEFT JOIN a_policy_and_sleds ON a_policy_and_sleds.sled_id = sled_targets.sled_id) + LEFT JOIN aa_policy_and_sleds ON aa_policy_and_sleds.sled_id = sled_targets.sled_id From c9bb4c7949828e9b0daa0533be88fe8af8d9b0ff Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 13:00:41 -0800 Subject: [PATCH 39/84] passing tests --- nexus/db-queries/src/db/datastore/sled.rs | 232 +++++++++--------- .../src/db/queries/sled_reservation.rs | 182 ++++++++++++++ .../tests/output/sled_find_targets_query.sql | 25 +- .../output/sled_insert_resource_query.sql | 104 ++++++++ 4 files changed, 428 insertions(+), 115 deletions(-) create mode 100644 nexus/db-queries/tests/output/sled_insert_resource_query.sql diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 0d3c82b6b12..ad48a5ce8cd 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -22,14 +22,13 @@ use crate::db::pagination::paginated; use crate::db::pagination::Paginator; use crate::db::pool::DbConnection; use crate::db::queries::sled_reservation::sled_find_targets_query; +use crate::db::queries::sled_reservation::sled_insert_resource_query; use crate::db::update_and_check::{UpdateAndCheck, UpdateStatus}; use crate::db::TransactionError; use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; -use itertools::Either; -use itertools::Itertools; use nexus_db_model::ApplySledFilterExt; use nexus_types::deployment::SledFilter; use nexus_types::external_api::views::SledPolicy; @@ -206,23 +205,12 @@ enum SledReservationTransactionError { // - Fail, no targets are available. fn pick_sled_reservation_target( log: &Logger, - mut targets: HashSet, - anti_affinity_sleds: Vec<(AffinityPolicy, SledUuid)>, - affinity_sleds: Vec<(AffinityPolicy, SledUuid)>, + targets: &HashSet, + banned: &HashSet, + unpreferred: &HashSet, + required: &HashSet, + preferred: &HashSet, ) -> Result { - let (banned, mut unpreferred): (HashSet<_>, HashSet<_>) = - anti_affinity_sleds.into_iter().partition_map(|(policy, id)| { - match policy { - AffinityPolicy::Fail => Either::Left(id), - AffinityPolicy::Allow => Either::Right(id), - } - }); - let (required, mut preferred): (HashSet<_>, HashSet<_>) = - affinity_sleds.into_iter().partition_map(|(policy, id)| match policy { - AffinityPolicy::Fail => Either::Left(id), - AffinityPolicy::Allow => Either::Right(id), - }); - if !banned.is_empty() { info!( log, @@ -255,12 +243,15 @@ fn pick_sled_reservation_target( } // We have no "required" sleds, but might have preferences. - targets = targets.difference(&banned).cloned().collect(); + let mut targets: HashSet<_> = + targets.difference(&banned).cloned().collect(); // Only consider "preferred" sleds that are viable targets - preferred = targets.intersection(&preferred).cloned().collect(); + let preferred: HashSet<_> = + targets.intersection(&preferred).cloned().collect(); // Only consider "unpreferred" sleds that are viable targets - unpreferred = targets.intersection(&unpreferred).cloned().collect(); + let mut unpreferred: HashSet<_> = + targets.intersection(&unpreferred).cloned().collect(); // If a target is both preferred and unpreferred, it is not considered // a part of either set. @@ -455,9 +446,24 @@ impl DataStore { constraints: db::model::SledReservationConstraints, ) -> Result { - let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; + // Check if resource ID already exists - if so, return it. + // + // This check makes this function idempotent. Beyond this point, however + // we rely on primary key constraints in the database to prevent + // concurrent reservations for same propolis_id. + use db::schema::sled_resource_vmm::dsl as resource_dsl; + let old_resource = resource_dsl::sled_resource_vmm + .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) + .select(SledResourceVmm::as_select()) + .limit(1) + .load_async(&*conn) + .await?; + if !old_resource.is_empty() { + return Ok(old_resource[0].clone()); + } + let must_use_sleds: HashSet = constraints .must_select_from() .into_iter() @@ -465,97 +471,103 @@ impl DataStore { .map(|id| SledUuid::from_untyped_uuid(*id)) .collect(); - self.transaction_retry_wrapper("sled_reservation_create") - .transaction(&conn, |conn| { - // Clone variables into retryable function - let err = err.clone(); - let must_use_sleds = must_use_sleds.clone(); - let resources = resources.clone(); + // Query for the set of possible sleds using a CTE. + // + // Note that this is not transactional, to reduce contention. + // However, that lack of transactionality means we need to validate + // our constraints again when we later try to INSERT the reservation. + let possible_sleds = sled_find_targets_query(instance_id, &resources) + .get_results_async::<( + // Sled UUID + Uuid, + // Would an allocation to this sled fit? + bool, + // Affinity policy on this sled + Option, + // Anti-affinity policy on this sled + Option, + )>(&*conn).await?; + + // Translate the database results into a format which we can use to pick + // a sled using more complex rules. + // + // See: `pick_sled_reservation_target(...)` + let mut sled_targets = HashSet::new(); + let mut banned = HashSet::new(); + let mut unpreferred = HashSet::new(); + let mut required = HashSet::new(); + let mut preferred = HashSet::new(); + for (sled_id, fits, affinity_policy, anti_affinity_policy) in + possible_sleds + { + let sled_id = SledUuid::from_untyped_uuid(sled_id); - async move { - use db::schema::sled_resource_vmm::dsl as resource_dsl; - - // Check if resource ID already exists - if so, return it. - let old_resource = resource_dsl::sled_resource_vmm - .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) - .select(SledResourceVmm::as_select()) - .limit(1) - .load_async(&conn) - .await?; - - if !old_resource.is_empty() { - return Ok(old_resource[0].clone()); - } + if fits + && (must_use_sleds.is_empty() + || must_use_sleds.contains(&sled_id)) + { + sled_targets.insert(sled_id); + } + if let Some(policy) = affinity_policy { + match policy { + AffinityPolicy::Fail => required.insert(sled_id), + AffinityPolicy::Allow => preferred.insert(sled_id), + }; + } + if let Some(policy) = anti_affinity_policy { + match policy { + AffinityPolicy::Fail => banned.insert(sled_id), + AffinityPolicy::Allow => unpreferred.insert(sled_id), + }; + } + } - let possible_sleds = sled_find_targets_query(instance_id, &resources) - .get_results_async::<( - // Sled UUID - Uuid, - // Would an allocation to this sled fit? - bool, - // Affinity policy on this sled - Option, - // Anti-affinity policy on this sled - Option, - )>(&conn).await?; - - println!("Observed sleds: {possible_sleds:#?}"); - - let mut sled_targets = HashSet::new(); - let mut affinity_sleds = vec![]; - let mut anti_affinity_sleds = vec![]; - - for ( - sled_id, - fits, - affinity_policy, - anti_affinity_policy, - ) in possible_sleds { - let sled_id = SledUuid::from_untyped_uuid(sled_id); - - if fits && (must_use_sleds.is_empty() || must_use_sleds.contains(&sled_id)) { - sled_targets.insert(sled_id); - } - if let Some(policy) = affinity_policy { - affinity_sleds.push((policy, sled_id)); - } - if let Some(policy) = anti_affinity_policy { - anti_affinity_sleds.push((policy, sled_id)); - } - } + // TODO: are we picking targets randomly enough?? + // maybe this needs to be a property of picking a sled reservation + // target. - let sled_target = pick_sled_reservation_target( - &opctx.log, - sled_targets, - anti_affinity_sleds, - affinity_sleds, - ).map_err(|e| { - err.bail(e) - })?; - - // Create a SledResourceVmm record, associate it with the target - // sled. - let resource = SledResourceVmm::new( - propolis_id, - instance_id, - sled_target, - resources, - ); - - diesel::insert_into(resource_dsl::sled_resource_vmm) - .values(resource) - .returning(SledResourceVmm::as_returning()) - .get_result_async(&conn) - .await - } - }) - .await - .map_err(|e| { - if let Some(err) = err.take() { - return SledReservationTransactionError::Reservation(err); - } - SledReservationTransactionError::Diesel(e) - }) + // We loop here because our attempts to INSERT may be violated by + // concurrent operations. We'll respond by looking through a slightly + // smaller set of possible sleds. + // + // In the uncontended case, however, we'll only iterate through this + // loop once. + loop { + // Pick a reservation target, given the constraints we previously + // saw in the database. + let sled_target = pick_sled_reservation_target( + &opctx.log, + &sled_targets, + &banned, + &unpreferred, + &required, + &preferred, + )?; + + // Create a SledResourceVmm record, associate it with the target + // sled. + let resource = SledResourceVmm::new( + propolis_id, + instance_id, + sled_target, + resources.clone(), + ); + + // Try to INSERT the record. If this is still a valid target, we'll + // use it. If it isn't a valid target, we'll shrink the set of + // viable sled targets and try again. + let rows_inserted = sled_insert_resource_query(&resource) + .execute_async(&*conn) + .await?; + if rows_inserted > 0 { + return Ok(resource); + } + + sled_targets.remove(&sled_target); + banned.remove(&sled_target); + unpreferred.remove(&sled_target); + preferred.remove(&sled_target); + } } pub async fn sled_reservation_delete( diff --git a/nexus/db-queries/src/db/queries/sled_reservation.rs b/nexus/db-queries/src/db/queries/sled_reservation.rs index b5ba7ba880b..7c0c92e2f74 100644 --- a/nexus/db-queries/src/db/queries/sled_reservation.rs +++ b/nexus/db-queries/src/db/queries/sled_reservation.rs @@ -6,6 +6,7 @@ use crate::db::model::AffinityPolicyEnum; use crate::db::model::Resources; +use crate::db::model::SledResourceVmm; use crate::db::raw_query_builder::QueryBuilder; use crate::db::raw_query_builder::TypedSqlQuery; use diesel::sql_types; @@ -146,6 +147,128 @@ pub fn sled_find_targets_query( .query() } +/// Inserts a sled_resource_vmm record into the database, if it is +/// a valid reservation. +pub fn sled_insert_resource_query( + resource: &SledResourceVmm, +) -> TypedSqlQuery<(sql_types::Numeric,)> { + QueryBuilder::new().sql(" + WITH sled_has_space AS ( + SELECT 1 + FROM sled + LEFT JOIN sled_resource_vmm + ON sled_resource_vmm.sled_id = sled.id + WHERE + sled.id = ").param().sql(" AND + sled.time_deleted IS NULL AND + sled.sled_policy = 'in_service' AND + sled.sled_state = 'active' + GROUP BY sled.id + HAVING + COALESCE(SUM(CAST(sled_resource_vmm.hardware_threads AS INT8)), 0) + " + ).param().sql(" <= sled.usable_hardware_threads AND + COALESCE(SUM(CAST(sled_resource_vmm.rss_ram AS INT8)), 0) + " + ).param().sql(" <= sled.usable_physical_ram AND + COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " + ).param().sql(" <= sled.reservoir_size + ), + our_aa_groups AS ( + SELECT group_id + FROM anti_affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_aa_instances AS ( + SELECT anti_affinity_group_instance_membership.group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + WHERE instance_id != ").param().sql(" + ), + banned_instances AS ( + SELECT instance_id + FROM other_aa_instances + JOIN anti_affinity_group + ON + anti_affinity_group.id = other_aa_instances.group_id AND + anti_affinity_group.failure_domain = 'sled' AND + anti_affinity_group.policy = 'fail' + WHERE anti_affinity_group.time_deleted IS NULL + ), + banned_sleds AS ( + SELECT DISTINCT sled_id + FROM banned_instances + JOIN sled_resource_vmm + ON + sled_resource_vmm.instance_id = banned_instances.instance_id + ), + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), + required_instances AS ( + SELECT policy,instance_id + FROM other_a_instances + JOIN affinity_group + ON + affinity_group.id = other_a_instances.group_id AND + affinity_group.failure_domain = 'sled' AND + affinity_group.policy = 'fail' + WHERE affinity_group.time_deleted IS NULL + ), + required_sleds AS ( + SELECT DISTINCT sled_id + FROM required_instances + JOIN sled_resource_vmm + ON + sled_resource_vmm.instance_id = required_instances.instance_id + ), + insert_valid AS ( + SELECT 1 + WHERE + EXISTS(SELECT 1 FROM sled_has_space) AND + NOT(EXISTS(SELECT 1 FROM banned_sleds WHERE sled_id = ").param().sql(")) AND + ( + EXISTS(SELECT 1 FROM required_sleds WHERE sled_id = ").param().sql(") OR + NOT EXISTS (SELECT 1 FROM required_sleds) + ) + ) + INSERT INTO sled_resource_vmm (id, sled_id, hardware_threads, rss_ram, reservoir_ram, instance_id) + SELECT + ").param().sql(", + ").param().sql(", + ").param().sql(", + ").param().sql(", + ").param().sql(", + ").param().sql(" + WHERE EXISTS(SELECT 1 FROM insert_valid) + ") + .bind::(resource.sled_id.into_untyped_uuid()) + .bind::(resource.resources.hardware_threads) + .bind::(resource.resources.rss_ram) + .bind::(resource.resources.reservoir_ram) + .bind::(resource.instance_id.unwrap().into_untyped_uuid()) + .bind::(resource.instance_id.unwrap().into_untyped_uuid()) + .bind::(resource.instance_id.unwrap().into_untyped_uuid()) + .bind::(resource.instance_id.unwrap().into_untyped_uuid()) + .bind::(resource.sled_id.into_untyped_uuid()) + .bind::(resource.sled_id.into_untyped_uuid()) + .bind::(resource.id.into_untyped_uuid()) + .bind::(resource.sled_id.into_untyped_uuid()) + .bind::(resource.resources.hardware_threads) + .bind::(resource.resources.rss_ram) + .bind::(resource.resources.reservoir_ram) + .bind::(resource.instance_id.unwrap().into_untyped_uuid()) + .query() +} + #[cfg(test)] mod test { use super::*; @@ -156,6 +279,8 @@ mod test { use omicron_common::api::external; use omicron_test_utils::dev; + use omicron_uuid_kinds::PropolisUuid; + use omicron_uuid_kinds::SledUuid; #[tokio::test] async fn expectorate_sled_find_targets_query() { @@ -198,5 +323,62 @@ mod test { logctx.cleanup_successful(); } + #[tokio::test] + async fn expectorate_sled_insert_resource_query() { + let resource = SledResourceVmm::new( + PropolisUuid::nil(), + InstanceUuid::nil(), + SledUuid::nil(), + Resources::new( + 0, + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + ), + ); + + let query = sled_insert_resource_query(&resource); + expectorate_query_contents( + &query, + "tests/output/sled_insert_resource_query.sql", + ) + .await; + } + + #[tokio::test] + async fn explain_sled_insert_resource_query() { + let logctx = dev::test_setup_log("explain_sled_insert_resource_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let resource = SledResourceVmm::new( + PropolisUuid::nil(), + InstanceUuid::nil(), + SledUuid::nil(), + Resources::new( + 0, + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + ), + ); + + let query = sled_insert_resource_query(&resource); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.terminate().await; + logctx.cleanup_successful(); + } + // NOTE: There are some tests you might wanna nab from "affinity.rs" } diff --git a/nexus/db-queries/tests/output/sled_find_targets_query.sql b/nexus/db-queries/tests/output/sled_find_targets_query.sql index ca9e26bbcba..9de50f47f7f 100644 --- a/nexus/db-queries/tests/output/sled_find_targets_query.sql +++ b/nexus/db-queries/tests/output/sled_find_targets_query.sql @@ -82,9 +82,24 @@ WITH other_a_instances_by_policy JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_a_instances_by_policy.instance_id + ), + sleds_with_space + AS ( + SELECT + s.sled_id, a.policy AS a_policy, aa.policy AS aa_policy + FROM + sled_targets AS s + LEFT JOIN a_policy_and_sleds AS a ON a.sled_id = s.sled_id + LEFT JOIN aa_policy_and_sleds AS aa ON aa.sled_id = s.sled_id + ), + sleds_without_space + AS ( + SELECT + sled_id, policy AS a_policy, NULL AS aa_policy + FROM + a_policy_and_sleds + WHERE + a_policy_and_sleds.sled_id NOT IN (SELECT sled_id FROM sleds_with_space) ) -SELECT - a_policy_and_sleds.policy, aa_policy_and_sleds.policy, sled_targets.sled_id -FROM - (sled_targets LEFT JOIN a_policy_and_sleds ON a_policy_and_sleds.sled_id = sled_targets.sled_id) - LEFT JOIN aa_policy_and_sleds ON aa_policy_and_sleds.sled_id = sled_targets.sled_id +SELECT sled_id, true, a_policy, aa_policy FROM sleds_with_space +UNION SELECT sled_id, false, a_policy, aa_policy FROM sleds_without_space diff --git a/nexus/db-queries/tests/output/sled_insert_resource_query.sql b/nexus/db-queries/tests/output/sled_insert_resource_query.sql new file mode 100644 index 00000000000..c9d5ea6e0ed --- /dev/null +++ b/nexus/db-queries/tests/output/sled_insert_resource_query.sql @@ -0,0 +1,104 @@ +WITH + sled_has_space + AS ( + SELECT + sled.id AS sled_id + FROM + sled LEFT JOIN sled_resource_vmm ON sled_resource_vmm.sled_id = sled.id + WHERE + sled.id = $1 + AND sled.time_deleted IS NULL + AND sled.sled_policy = 'in_service' + AND sled.sled_state = 'active' + GROUP BY + sled.id + HAVING + COALESCE(sum(CAST(sled_resource_vmm.hardware_threads AS INT8)), 0) + $2 + <= sled.usable_hardware_threads + AND COALESCE(sum(CAST(sled_resource_vmm.rss_ram AS INT8)), 0) + $3 + <= sled.usable_physical_ram + AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $4 + <= sled.reservoir_size + ), + our_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $5), + other_aa_instances + AS ( + SELECT + anti_affinity_group_instance_membership.group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + WHERE + instance_id != $6 + ), + banned_instances + AS ( + SELECT + instance_id + FROM + other_aa_instances + JOIN anti_affinity_group ON + anti_affinity_group.id = other_aa_instances.group_id + AND anti_affinity_group.failure_domain = 'sled' + AND anti_affinity_group.policy = 'fail' + WHERE + anti_affinity_group.time_deleted IS NULL + ), + banned_sleds + AS ( + SELECT + DISTINCT sled_id + FROM + banned_instances + JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = banned_instances.instance_id + ), + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $7), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $8 + ), + required_instances + AS ( + SELECT + policy, instance_id + FROM + other_a_instances + JOIN affinity_group ON + affinity_group.id = other_a_instances.group_id + AND affinity_group.failure_domain = 'sled' + AND affinity_group.policy = 'fail' + WHERE + affinity_group.time_deleted IS NULL + ), + required_sleds + AS ( + SELECT + DISTINCT sled_id + FROM + required_instances + JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = required_instances.instance_id + ), + insert_valid + AS ( + SELECT + 1 + WHERE + EXISTS(SELECT 1 FROM sled_has_space) + AND NOT (EXISTS(SELECT $9 FROM banned_sleds)) + AND (EXISTS(SELECT $10 FROM required_sleds) OR NOT EXISTS(SELECT 1 FROM required_sleds)) + ) +INSERT +INTO + sled_resource_vmm (id, sled_id, hardware_threads, rss_ram, reservoir_ram, instance_id) +SELECT + $11, $12, $13, $14, $15, $16 +WHERE + EXISTS(SELECT 1 FROM insert_valid) From 8e5b1b0daee0176ea7ce4370312c2d3ffde08e1b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 13:09:21 -0800 Subject: [PATCH 40/84] cleanup --- nexus/db-queries/src/db/datastore/sled.rs | 4 - nexus/db-queries/src/db/queries/affinity.rs | 611 -------------------- nexus/db-queries/src/db/queries/mod.rs | 1 - 3 files changed, 616 deletions(-) delete mode 100644 nexus/db-queries/src/db/queries/affinity.rs diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index ad48a5ce8cd..566b025e851 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -522,10 +522,6 @@ impl DataStore { } } - // TODO: are we picking targets randomly enough?? - // maybe this needs to be a property of picking a sled reservation - // target. - // We loop here because our attempts to INSERT may be violated by // concurrent operations. We'll respond by looking through a slightly // smaller set of possible sleds. diff --git a/nexus/db-queries/src/db/queries/affinity.rs b/nexus/db-queries/src/db/queries/affinity.rs deleted file mode 100644 index 7ba2e107fee..00000000000 --- a/nexus/db-queries/src/db/queries/affinity.rs +++ /dev/null @@ -1,611 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Implementation of queries for affinity groups - -use crate::db::model::AffinityPolicyEnum; -use crate::db::raw_query_builder::{QueryBuilder, TypedSqlQuery}; -use diesel::sql_types; -use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::InstanceUuid; - -/// For an instance, look up all anti-affinity groups it belongs to. -/// For all those groups, find all instances with reservations, and look -/// up their sleds. -/// Return all the sleds on which those instances were found, along with -/// policy information about the corresponding group. -pub fn lookup_anti_affinity_sleds_query( - instance_id: InstanceUuid, -) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { - QueryBuilder::new().sql( - "WITH our_groups AS ( - SELECT group_id - FROM anti_affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_instances AS ( - SELECT anti_affinity_group_instance_membership.group_id,instance_id - FROM anti_affinity_group_instance_membership - JOIN our_groups - ON anti_affinity_group_instance_membership.group_id = our_groups.group_id - WHERE instance_id != ").param().sql(" - ), - other_instances_by_policy AS ( - SELECT policy,instance_id - FROM other_instances - JOIN anti_affinity_group - ON - anti_affinity_group.id = other_instances.group_id AND - anti_affinity_group.failure_domain = 'sled' - WHERE anti_affinity_group.time_deleted IS NULL - ) - SELECT DISTINCT policy,sled_id - FROM other_instances_by_policy - JOIN sled_resource_vmm - ON - sled_resource_vmm.instance_id = other_instances_by_policy.instance_id") - .bind::(instance_id.into_untyped_uuid()) - .bind::(instance_id.into_untyped_uuid()) - .query() -} - -/// For an instance, look up all affinity groups it belongs to. -/// For all those groups, find all instances with reservations, and look -/// up their sleds. -/// Return all the sleds on which those instances were found, along with -/// policy information about the corresponding group. -pub fn lookup_affinity_sleds_query( - instance_id: InstanceUuid, -) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { - QueryBuilder::new() - .sql( - "WITH our_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ", - ) - .param() - .sql( - " - ), - other_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_groups - ON affinity_group_instance_membership.group_id = our_groups.group_id - WHERE instance_id != ", - ) - .param() - .sql( - " - ), - other_instances_by_policy AS ( - SELECT policy,instance_id - FROM other_instances - JOIN affinity_group - ON - affinity_group.id = other_instances.group_id AND - affinity_group.failure_domain = 'sled' - WHERE affinity_group.time_deleted IS NULL - ) - SELECT DISTINCT policy,sled_id - FROM other_instances_by_policy - JOIN sled_resource_vmm - ON - sled_resource_vmm.instance_id = other_instances_by_policy.instance_id", - ) - .bind::(instance_id.into_untyped_uuid()) - .bind::(instance_id.into_untyped_uuid()) - .query() -} - -#[cfg(test)] -mod test { - use super::*; - use crate::db::explain::ExplainableAsync; - use crate::db::model; - use crate::db::pub_test_utils::TestDatabase; - use crate::db::raw_query_builder::expectorate_query_contents; - - use anyhow::Context; - use async_bb8_diesel::AsyncRunQueryDsl; - use nexus_types::external_api::params; - use nexus_types::identity::Resource; - use omicron_common::api::external; - use omicron_test_utils::dev; - use omicron_uuid_kinds::AffinityGroupUuid; - use omicron_uuid_kinds::AntiAffinityGroupUuid; - use omicron_uuid_kinds::PropolisUuid; - use omicron_uuid_kinds::SledUuid; - use uuid::Uuid; - - #[tokio::test] - async fn expectorate_lookup_anti_affinity_sleds_query() { - let id = InstanceUuid::nil(); - - let query = lookup_anti_affinity_sleds_query(id); - expectorate_query_contents( - &query, - "tests/output/lookup_anti_affinity_sleds_query.sql", - ) - .await; - } - - #[tokio::test] - async fn expectorate_lookup_affinity_sleds_query() { - let id = InstanceUuid::nil(); - - let query = lookup_affinity_sleds_query(id); - expectorate_query_contents( - &query, - "tests/output/lookup_affinity_sleds_query.sql", - ) - .await; - } - - #[tokio::test] - async fn explain_lookup_anti_affinity_sleds_query() { - let logctx = - dev::test_setup_log("explain_lookup_anti_affinity_sleds_query"); - let db = TestDatabase::new_with_pool(&logctx.log).await; - let pool = db.pool(); - let conn = pool.claim().await.unwrap(); - - let id = InstanceUuid::nil(); - let query = lookup_anti_affinity_sleds_query(id); - let _ = query - .explain_async(&conn) - .await - .expect("Failed to explain query - is it valid SQL?"); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn explain_lookup_affinity_sleds_query() { - let logctx = dev::test_setup_log("explain_lookup_affinity_sleds_query"); - let db = TestDatabase::new_with_pool(&logctx.log).await; - let pool = db.pool(); - let conn = pool.claim().await.unwrap(); - - let id = InstanceUuid::nil(); - let query = lookup_affinity_sleds_query(id); - let _ = query - .explain_async(&conn) - .await - .expect("Failed to explain query - is it valid SQL?"); - - db.terminate().await; - logctx.cleanup_successful(); - } - - async fn make_affinity_group( - project_id: Uuid, - name: &'static str, - policy: external::AffinityPolicy, - conn: &async_bb8_diesel::Connection, - ) -> anyhow::Result { - let group = model::AffinityGroup::new( - project_id, - params::AffinityGroupCreate { - identity: external::IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "desc".to_string(), - }, - policy, - failure_domain: external::FailureDomain::Sled, - }, - ); - use crate::db::schema::affinity_group::dsl; - diesel::insert_into(dsl::affinity_group) - .values(group.clone()) - .execute_async(conn) - .await - .context("Cannot create affinity group")?; - Ok(group) - } - - async fn make_anti_affinity_group( - project_id: Uuid, - name: &'static str, - policy: external::AffinityPolicy, - conn: &async_bb8_diesel::Connection, - ) -> anyhow::Result { - let group = model::AntiAffinityGroup::new( - project_id, - params::AntiAffinityGroupCreate { - identity: external::IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "desc".to_string(), - }, - policy, - failure_domain: external::FailureDomain::Sled, - }, - ); - use crate::db::schema::anti_affinity_group::dsl; - diesel::insert_into(dsl::anti_affinity_group) - .values(group.clone()) - .execute_async(conn) - .await - .context("Cannot create anti affinity group")?; - Ok(group) - } - - async fn make_affinity_group_instance_membership( - group_id: AffinityGroupUuid, - instance_id: InstanceUuid, - conn: &async_bb8_diesel::Connection, - ) -> anyhow::Result<()> { - // Let's claim an instance belongs to that group - let membership = - model::AffinityGroupInstanceMembership::new(group_id, instance_id); - use crate::db::schema::affinity_group_instance_membership::dsl; - diesel::insert_into(dsl::affinity_group_instance_membership) - .values(membership) - .execute_async(conn) - .await - .context("Cannot create affinity group instance membership")?; - Ok(()) - } - - async fn make_anti_affinity_group_instance_membership( - group_id: AntiAffinityGroupUuid, - instance_id: InstanceUuid, - conn: &async_bb8_diesel::Connection, - ) -> anyhow::Result<()> { - // Let's claim an instance belongs to that group - let membership = model::AntiAffinityGroupInstanceMembership::new( - group_id, - instance_id, - ); - use crate::db::schema::anti_affinity_group_instance_membership::dsl; - diesel::insert_into(dsl::anti_affinity_group_instance_membership) - .values(membership) - .execute_async(conn) - .await - .context("Cannot create anti affinity group instance membership")?; - Ok(()) - } - - async fn make_sled_resource_vmm( - sled_id: SledUuid, - instance_id: InstanceUuid, - conn: &async_bb8_diesel::Connection, - ) -> anyhow::Result<()> { - let resource = model::SledResourceVmm::new( - PropolisUuid::new_v4(), - instance_id, - sled_id, - model::Resources::new( - 0, - model::ByteCount::from( - external::ByteCount::from_gibibytes_u32(0), - ), - model::ByteCount::from( - external::ByteCount::from_gibibytes_u32(0), - ), - ), - ); - use crate::db::schema::sled_resource_vmm::dsl; - diesel::insert_into(dsl::sled_resource_vmm) - .values(resource) - .execute_async(conn) - .await - .context("Cannot create sled resource")?; - Ok(()) - } - - #[tokio::test] - async fn lookup_affinity_sleds() { - let logctx = dev::test_setup_log("lookup_affinity_sleds"); - let db = TestDatabase::new_with_pool(&logctx.log).await; - let pool = db.pool(); - let conn = pool.claim().await.unwrap(); - - let our_instance_id = InstanceUuid::new_v4(); - let other_instance_id = InstanceUuid::new_v4(); - - // With no groups and no instances, we should see no other instances - // belonging to our affinity group. - assert_eq!( - lookup_affinity_sleds_query(our_instance_id,) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(), - vec![], - ); - - let sled_id = SledUuid::new_v4(); - let project_id = Uuid::new_v4(); - - // Make a group, add our instance to it - let group = make_affinity_group( - project_id, - "group-allow", - external::AffinityPolicy::Allow, - &conn, - ) - .await - .unwrap(); - make_affinity_group_instance_membership( - AffinityGroupUuid::from_untyped_uuid(group.id()), - our_instance_id, - &conn, - ) - .await - .unwrap(); - - // Create an instance which belongs to that group - make_affinity_group_instance_membership( - AffinityGroupUuid::from_untyped_uuid(group.id()), - other_instance_id, - &conn, - ) - .await - .unwrap(); - - make_sled_resource_vmm(sled_id, other_instance_id, &conn) - .await - .unwrap(); - - // Now if we look, we'll find the "other sled", on which the "other - // instance" was placed. - assert_eq!( - lookup_affinity_sleds_query(our_instance_id,) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(), - vec![(model::AffinityPolicy::Allow, sled_id.into_untyped_uuid())], - ); - - // If we look from the perspective of the other instance, - // we "ignore ourselves" for placement, so the set of sleds to consider - // is still empty. - assert_eq!( - lookup_affinity_sleds_query(other_instance_id,) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(), - vec![] - ); - - // If we make another group (note the policy is different this time!) - // it'll also be reflected in the results. - let group = make_affinity_group( - project_id, - "group-fail", - external::AffinityPolicy::Fail, - &conn, - ) - .await - .unwrap(); - make_affinity_group_instance_membership( - AffinityGroupUuid::from_untyped_uuid(group.id()), - our_instance_id, - &conn, - ) - .await - .unwrap(); - make_affinity_group_instance_membership( - AffinityGroupUuid::from_untyped_uuid(group.id()), - other_instance_id, - &conn, - ) - .await - .unwrap(); - - // We see the outcome of both groups, with a different policy for each. - let mut results = lookup_affinity_sleds_query(our_instance_id) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(); - results.sort(); - assert_eq!( - results, - vec![ - (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), - (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), - ], - ); - - // Let's add one more group, to see what happens to the query output. - let group = make_affinity_group( - project_id, - "group-fail2", - external::AffinityPolicy::Fail, - &conn, - ) - .await - .unwrap(); - make_affinity_group_instance_membership( - AffinityGroupUuid::from_untyped_uuid(group.id()), - our_instance_id, - &conn, - ) - .await - .unwrap(); - make_affinity_group_instance_membership( - AffinityGroupUuid::from_untyped_uuid(group.id()), - other_instance_id, - &conn, - ) - .await - .unwrap(); - let mut results = lookup_affinity_sleds_query(our_instance_id) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(); - results.sort(); - - // Since we use "SELECT DISTINCT", the results are bounded, and do not - // grow as we keep on finding more sleds that have the same policy. - assert_eq!( - results, - vec![ - (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), - (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), - ], - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn lookup_anti_affinity_sleds() { - let logctx = dev::test_setup_log("lookup_anti_affinity_sleds"); - let db = TestDatabase::new_with_pool(&logctx.log).await; - let pool = db.pool(); - let conn = pool.claim().await.unwrap(); - - let our_instance_id = InstanceUuid::new_v4(); - let other_instance_id = InstanceUuid::new_v4(); - - // With no groups and no instances, we should see no other instances - // belonging to our anti-affinity group. - assert_eq!( - lookup_anti_affinity_sleds_query(our_instance_id,) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(), - vec![], - ); - - let sled_id = SledUuid::new_v4(); - let project_id = Uuid::new_v4(); - - // Make a group, add our instance to it - let group = make_anti_affinity_group( - project_id, - "group-allow", - external::AffinityPolicy::Allow, - &conn, - ) - .await - .unwrap(); - make_anti_affinity_group_instance_membership( - AntiAffinityGroupUuid::from_untyped_uuid(group.id()), - our_instance_id, - &conn, - ) - .await - .unwrap(); - - // Create an instance which belongs to that group - make_anti_affinity_group_instance_membership( - AntiAffinityGroupUuid::from_untyped_uuid(group.id()), - other_instance_id, - &conn, - ) - .await - .unwrap(); - - make_sled_resource_vmm(sled_id, other_instance_id, &conn) - .await - .unwrap(); - - // Now if we look, we'll find the "other sled", on which the "other - // instance" was placed. - assert_eq!( - lookup_anti_affinity_sleds_query(our_instance_id,) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(), - vec![(model::AffinityPolicy::Allow, sled_id.into_untyped_uuid())], - ); - - // If we look from the perspective of the other instance, - // we "ignore ourselves" for placement, so the set of sleds to consider - // is still empty. - assert_eq!( - lookup_anti_affinity_sleds_query(other_instance_id,) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(), - vec![] - ); - - // If we make another group (note the policy is different this time!) - // it'll also be reflected in the results. - let group = make_anti_affinity_group( - project_id, - "group-fail", - external::AffinityPolicy::Fail, - &conn, - ) - .await - .unwrap(); - make_anti_affinity_group_instance_membership( - AntiAffinityGroupUuid::from_untyped_uuid(group.id()), - our_instance_id, - &conn, - ) - .await - .unwrap(); - make_anti_affinity_group_instance_membership( - AntiAffinityGroupUuid::from_untyped_uuid(group.id()), - other_instance_id, - &conn, - ) - .await - .unwrap(); - - // We see the outcome of both groups, with a different policy for each. - let mut results = lookup_anti_affinity_sleds_query(our_instance_id) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(); - results.sort(); - assert_eq!( - results, - vec![ - (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), - (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), - ], - ); - - // Let's add one more group, to see what happens to the query output. - let group = make_anti_affinity_group( - project_id, - "group-fail2", - external::AffinityPolicy::Fail, - &conn, - ) - .await - .unwrap(); - make_anti_affinity_group_instance_membership( - AntiAffinityGroupUuid::from_untyped_uuid(group.id()), - our_instance_id, - &conn, - ) - .await - .unwrap(); - make_anti_affinity_group_instance_membership( - AntiAffinityGroupUuid::from_untyped_uuid(group.id()), - other_instance_id, - &conn, - ) - .await - .unwrap(); - let mut results = lookup_anti_affinity_sleds_query(our_instance_id) - .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) - .await - .unwrap(); - results.sort(); - - // Since we use "SELECT DISTINCT", the results are bounded, and do not - // grow as we keep on finding more sleds that have the same policy. - assert_eq!( - results, - vec![ - (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), - (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), - ], - ); - - db.terminate().await; - logctx.cleanup_successful(); - } -} diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index 128c73896ad..4d3eb23f40b 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -5,7 +5,6 @@ //! Specialized queries for inserting database records, usually to maintain //! complex invariants that are most accurately expressed in a single query. -pub mod affinity; pub mod disk; pub mod external_ip; pub mod ip_pool; From d1356f8fa048f74eef05d57312b305b0f4876a4d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 15:18:46 -0800 Subject: [PATCH 41/84] Updated expectorate output --- .../tests/output/sled_insert_resource_query.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nexus/db-queries/tests/output/sled_insert_resource_query.sql b/nexus/db-queries/tests/output/sled_insert_resource_query.sql index c9d5ea6e0ed..9cfb68e3008 100644 --- a/nexus/db-queries/tests/output/sled_insert_resource_query.sql +++ b/nexus/db-queries/tests/output/sled_insert_resource_query.sql @@ -2,7 +2,7 @@ WITH sled_has_space AS ( SELECT - sled.id AS sled_id + 1 FROM sled LEFT JOIN sled_resource_vmm ON sled_resource_vmm.sled_id = sled.id WHERE @@ -92,8 +92,11 @@ WITH 1 WHERE EXISTS(SELECT 1 FROM sled_has_space) - AND NOT (EXISTS(SELECT $9 FROM banned_sleds)) - AND (EXISTS(SELECT $10 FROM required_sleds) OR NOT EXISTS(SELECT 1 FROM required_sleds)) + AND NOT (EXISTS(SELECT 1 FROM banned_sleds WHERE sled_id = $9)) + AND ( + EXISTS(SELECT 1 FROM required_sleds WHERE sled_id = $10) + OR NOT EXISTS(SELECT 1 FROM required_sleds) + ) ) INSERT INTO From 7abc2b47cb31ad504250b561d3631d42359e3cd6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 17:02:08 -0800 Subject: [PATCH 42/84] Testing contention more explicitly --- nexus/db-queries/src/db/datastore/sled.rs | 300 ++++++++++++++++++ .../src/db/queries/sled_reservation.rs | 3 +- 2 files changed, 302 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 566b025e851..86c0c5643da 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -1618,6 +1618,13 @@ pub(in crate::db::datastore) mod test { resources: db::model::Resources, } + struct FindTargetsOutput { + id: SledUuid, + fits: bool, + affinity_policy: Option, + anti_affinity_policy: Option, + } + impl Instance { fn new() -> Self { Self { @@ -1628,6 +1635,55 @@ pub(in crate::db::datastore) mod test { } } + // This is the first half of creating a sled reservation. + // It can be called during tests trying to invoke contention manually. + async fn find_targets( + &self, + datastore: &DataStore, + ) -> Vec { + assert!(self.force_onto_sled.is_none()); + + sled_find_targets_query(self.id, &self.resources) + .get_results_async::<( + Uuid, + bool, + Option, + Option, + )>(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap() + .into_iter() + .map(|(id, fits, affinity_policy, anti_affinity_policy)| { + FindTargetsOutput { id: SledUuid::from_untyped_uuid(id), fits, affinity_policy, anti_affinity_policy } + }) + .collect() + } + + // This is the second half of creating a sled reservation. + // It can be called during tests trying to invoke contention manually. + // + // Returns "true" if the INSERT succeeded + async fn insert_resource( + &self, + datastore: &DataStore, + propolis_id: PropolisUuid, + sled_id: SledUuid, + ) -> bool { + assert!(self.force_onto_sled.is_none()); + + let resource = SledResourceVmm::new( + propolis_id, + self.id, + sled_id, + self.resources.clone(), + ); + + sled_insert_resource_query(&resource) + .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap() > 0 + } + fn use_many_resources(mut self) -> Self { self.resources = large_resource_request(); self @@ -2445,6 +2501,250 @@ pub(in crate::db::datastore) mod test { logctx.cleanup_successful(); } + // Test that concurrent provisioning of an affinity group can cause + // the INSERT of a sled_resource_vmm to fail. + #[tokio::test] + async fn sled_reservation_concurrent_affinity_requirement() { + let logctx = dev::test_setup_log("sled_reservation_concurrent_affinity_requirement"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let test_instance = Instance::new().group("affinity"); + + // We manually call the first half of sled reservation: finding targets. + // + // All sleds should be available. + let possible_sleds = test_instance.find_targets(&datastore).await; + assert_eq!(possible_sleds.len(), SLED_COUNT); + assert!(possible_sleds.iter().all(|sled| sled.fits)); + assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); + assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + + // Concurrently create an instance on sleds[0]. + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new() + .group("affinity") + .sled(sleds[0].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + // Put the instance-under-test in the "affinity" group. + test_instance.add_to_groups(&datastore, &all_groups).await; + + // Now if we try to find targets again, the result will change. + let possible_sleds = test_instance.find_targets(&datastore).await; + assert_eq!(possible_sleds.len(), SLED_COUNT); + assert!(possible_sleds.iter().all(|sled| sled.fits)); + assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + let affine_sled = possible_sleds.iter() + .find(|sled| sled.id.into_untyped_uuid() == sleds[0].id()) + .unwrap(); + assert!( + matches!( + affine_sled.affinity_policy.expect("Sled 0 should be affine"), + AffinityPolicy::Fail + ) + ); + + // Inserting onto sleds[1..3] should fail -- the affinity requirement + // should bind us to sleds[0]. + for i in 1..=3 { + assert!(!test_instance.insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[i].id()), + ).await, "Shouldn't have been able to insert into sled {i}") + }; + + // Inserting into sleds[0] should succeed + assert!(test_instance.insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[0].id()), + ).await); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Test that concurrent provisioning of an anti-affinity group can cause + // the INSERT of a sled_resource_vmm to fail. + #[tokio::test] + async fn sled_reservation_concurrent_anti_affinity_requirement() { + let logctx = dev::test_setup_log("sled_reservation_concurrent_anti_affinity_requirement"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let test_instance = Instance::new().group("anti-affinity"); + + // We manually call the first half of sled reservation: finding targets. + // + // All sleds should be available. + let possible_sleds = test_instance.find_targets(&datastore).await; + assert_eq!(possible_sleds.len(), SLED_COUNT); + assert!(possible_sleds.iter().all(|sled| sled.fits)); + assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); + assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + + // Concurrently create an instance on sleds[0]. + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new() + .group("anti-affinity") + .sled(sleds[0].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + // Put the instance-under-test in the "anti-affinity" group. + test_instance.add_to_groups(&datastore, &all_groups).await; + + // Now if we try to find targets again, the result will change. + let possible_sleds = test_instance.find_targets(&datastore).await; + assert_eq!(possible_sleds.len(), SLED_COUNT); + assert!(possible_sleds.iter().all(|sled| sled.fits)); + assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); + let anti_affine_sled = possible_sleds.iter() + .find(|sled| sled.id.into_untyped_uuid() == sleds[0].id()) + .unwrap(); + assert!( + matches!( + anti_affine_sled.anti_affinity_policy.expect("Sled 0 should be anti-affine"), + AffinityPolicy::Fail + ) + ); + + // Inserting onto sleds[0] should fail -- the anti-affinity requirement + // should prevent us from inserting there. + assert!(!test_instance.insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[0].id()), + ).await, "Shouldn't have been able to insert into sleds[0]"); + + // Inserting into sleds[1] should succeed + assert!(test_instance.insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[1].id()), + ).await); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn sled_reservation_concurrent_space_requirement() { + let logctx = dev::test_setup_log("sled_reservation_concurrent_space_requirement"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let test_instance = Instance::new().use_many_resources(); + + // We manually call the first half of sled reservation: finding targets. + // + // All sleds should be available. + let possible_sleds = test_instance.find_targets(&datastore).await; + assert_eq!(possible_sleds.len(), SLED_COUNT); + assert!(possible_sleds.iter().all(|sled| sled.fits)); + assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); + assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + + // Concurrently create large instances on sleds 0, 2, 3. + let groups = []; + let all_groups = AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new() + .use_many_resources() + .sled(sleds[0].id()), + Instance::new() + .use_many_resources() + .sled(sleds[2].id()), + Instance::new() + .use_many_resources() + .sled(sleds[3].id()), + + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + // Now if we try to find targets again, the result will change. + let possible_sleds = test_instance.find_targets(&datastore).await; + assert_eq!(possible_sleds.len(), 1); + assert!(possible_sleds[0].affinity_policy.is_none()); + assert!(possible_sleds[0].anti_affinity_policy.is_none()); + assert!(possible_sleds[0].fits); + assert_eq!(possible_sleds[0].id.into_untyped_uuid(), sleds[1].id()); + + // Inserting onto sleds[0, 2, 3] should fail - there shouldn't + // be enough space on these sleds. + for i in [0, 2, 3] { + assert!(!test_instance.insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[i].id()), + ).await, "Shouldn't have been able to insert into sleds[i]"); + } + + // Inserting into sleds[1] should succeed + assert!(test_instance.insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[1].id()), + ).await); + + db.terminate().await; + logctx.cleanup_successful(); + } + async fn lookup_physical_disk( datastore: &DataStore, id: PhysicalDiskUuid, diff --git a/nexus/db-queries/src/db/queries/sled_reservation.rs b/nexus/db-queries/src/db/queries/sled_reservation.rs index 7c0c92e2f74..912fd644da4 100644 --- a/nexus/db-queries/src/db/queries/sled_reservation.rs +++ b/nexus/db-queries/src/db/queries/sled_reservation.rs @@ -380,5 +380,6 @@ mod test { logctx.cleanup_successful(); } - // NOTE: There are some tests you might wanna nab from "affinity.rs" + // NOTE: These queries are more exhaustively tested in db/datastore/sled.rs, + // where they are used. } From bd14e7b73c019e2baeebf8770b006848fbbf0145 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 12 Feb 2025 17:03:20 -0800 Subject: [PATCH 43/84] fmt --- nexus/db-queries/src/db/datastore/sled.rs | 223 +++++++++++++--------- 1 file changed, 129 insertions(+), 94 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 86c0c5643da..227f258c174 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -1679,9 +1679,12 @@ pub(in crate::db::datastore) mod test { ); sled_insert_resource_query(&resource) - .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await - .unwrap() > 0 + .unwrap() + > 0 } fn use_many_resources(mut self) -> Self { @@ -2505,7 +2508,9 @@ pub(in crate::db::datastore) mod test { // the INSERT of a sled_resource_vmm to fail. #[tokio::test] async fn sled_reservation_concurrent_affinity_requirement() { - let logctx = dev::test_setup_log("sled_reservation_concurrent_affinity_requirement"); + let logctx = dev::test_setup_log( + "sled_reservation_concurrent_affinity_requirement", + ); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = @@ -2522,25 +2527,23 @@ pub(in crate::db::datastore) mod test { let possible_sleds = test_instance.find_targets(&datastore).await; assert_eq!(possible_sleds.len(), SLED_COUNT); assert!(possible_sleds.iter().all(|sled| sled.fits)); - assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); - assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + assert!(possible_sleds + .iter() + .all(|sled| sled.affinity_policy.is_none())); + assert!(possible_sleds + .iter() + .all(|sled| sled.anti_affinity_policy.is_none())); // Concurrently create an instance on sleds[0]. - let groups = [ - Group { - affinity: Affinity::Positive, - name: "affinity", - policy: external::AffinityPolicy::Fail, - }, - ]; + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }]; let all_groups = AllGroups::create(&opctx, &datastore, &authz_project, &groups) .await; - let instances = [ - Instance::new() - .group("affinity") - .sled(sleds[0].id()), - ]; + let instances = [Instance::new().group("affinity").sled(sleds[0].id())]; for instance in instances { instance .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) @@ -2555,33 +2558,43 @@ pub(in crate::db::datastore) mod test { let possible_sleds = test_instance.find_targets(&datastore).await; assert_eq!(possible_sleds.len(), SLED_COUNT); assert!(possible_sleds.iter().all(|sled| sled.fits)); - assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); - let affine_sled = possible_sleds.iter() + assert!(possible_sleds + .iter() + .all(|sled| sled.anti_affinity_policy.is_none())); + let affine_sled = possible_sleds + .iter() .find(|sled| sled.id.into_untyped_uuid() == sleds[0].id()) .unwrap(); - assert!( - matches!( - affine_sled.affinity_policy.expect("Sled 0 should be affine"), - AffinityPolicy::Fail - ) - ); + assert!(matches!( + affine_sled.affinity_policy.expect("Sled 0 should be affine"), + AffinityPolicy::Fail + )); // Inserting onto sleds[1..3] should fail -- the affinity requirement // should bind us to sleds[0]. for i in 1..=3 { - assert!(!test_instance.insert_resource( - &datastore, - PropolisUuid::new_v4(), - SledUuid::from_untyped_uuid(sleds[i].id()), - ).await, "Shouldn't have been able to insert into sled {i}") - }; + assert!( + !test_instance + .insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[i].id()), + ) + .await, + "Shouldn't have been able to insert into sled {i}" + ) + } // Inserting into sleds[0] should succeed - assert!(test_instance.insert_resource( - &datastore, - PropolisUuid::new_v4(), - SledUuid::from_untyped_uuid(sleds[0].id()), - ).await); + assert!( + test_instance + .insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[0].id()), + ) + .await + ); db.terminate().await; logctx.cleanup_successful(); @@ -2591,7 +2604,9 @@ pub(in crate::db::datastore) mod test { // the INSERT of a sled_resource_vmm to fail. #[tokio::test] async fn sled_reservation_concurrent_anti_affinity_requirement() { - let logctx = dev::test_setup_log("sled_reservation_concurrent_anti_affinity_requirement"); + let logctx = dev::test_setup_log( + "sled_reservation_concurrent_anti_affinity_requirement", + ); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = @@ -2608,25 +2623,24 @@ pub(in crate::db::datastore) mod test { let possible_sleds = test_instance.find_targets(&datastore).await; assert_eq!(possible_sleds.len(), SLED_COUNT); assert!(possible_sleds.iter().all(|sled| sled.fits)); - assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); - assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + assert!(possible_sleds + .iter() + .all(|sled| sled.affinity_policy.is_none())); + assert!(possible_sleds + .iter() + .all(|sled| sled.anti_affinity_policy.is_none())); // Concurrently create an instance on sleds[0]. - let groups = [ - Group { - affinity: Affinity::Negative, - name: "anti-affinity", - policy: external::AffinityPolicy::Fail, - }, - ]; + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }]; let all_groups = AllGroups::create(&opctx, &datastore, &authz_project, &groups) .await; - let instances = [ - Instance::new() - .group("anti-affinity") - .sled(sleds[0].id()), - ]; + let instances = + [Instance::new().group("anti-affinity").sled(sleds[0].id())]; for instance in instances { instance .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) @@ -2641,31 +2655,43 @@ pub(in crate::db::datastore) mod test { let possible_sleds = test_instance.find_targets(&datastore).await; assert_eq!(possible_sleds.len(), SLED_COUNT); assert!(possible_sleds.iter().all(|sled| sled.fits)); - assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); - let anti_affine_sled = possible_sleds.iter() + assert!(possible_sleds + .iter() + .all(|sled| sled.affinity_policy.is_none())); + let anti_affine_sled = possible_sleds + .iter() .find(|sled| sled.id.into_untyped_uuid() == sleds[0].id()) .unwrap(); - assert!( - matches!( - anti_affine_sled.anti_affinity_policy.expect("Sled 0 should be anti-affine"), - AffinityPolicy::Fail - ) - ); + assert!(matches!( + anti_affine_sled + .anti_affinity_policy + .expect("Sled 0 should be anti-affine"), + AffinityPolicy::Fail + )); // Inserting onto sleds[0] should fail -- the anti-affinity requirement // should prevent us from inserting there. - assert!(!test_instance.insert_resource( - &datastore, - PropolisUuid::new_v4(), - SledUuid::from_untyped_uuid(sleds[0].id()), - ).await, "Shouldn't have been able to insert into sleds[0]"); + assert!( + !test_instance + .insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[0].id()), + ) + .await, + "Shouldn't have been able to insert into sleds[0]" + ); // Inserting into sleds[1] should succeed - assert!(test_instance.insert_resource( - &datastore, - PropolisUuid::new_v4(), - SledUuid::from_untyped_uuid(sleds[1].id()), - ).await); + assert!( + test_instance + .insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[1].id()), + ) + .await + ); db.terminate().await; logctx.cleanup_successful(); @@ -2673,7 +2699,9 @@ pub(in crate::db::datastore) mod test { #[tokio::test] async fn sled_reservation_concurrent_space_requirement() { - let logctx = dev::test_setup_log("sled_reservation_concurrent_space_requirement"); + let logctx = dev::test_setup_log( + "sled_reservation_concurrent_space_requirement", + ); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = @@ -2690,24 +2718,22 @@ pub(in crate::db::datastore) mod test { let possible_sleds = test_instance.find_targets(&datastore).await; assert_eq!(possible_sleds.len(), SLED_COUNT); assert!(possible_sleds.iter().all(|sled| sled.fits)); - assert!(possible_sleds.iter().all(|sled| sled.affinity_policy.is_none())); - assert!(possible_sleds.iter().all(|sled| sled.anti_affinity_policy.is_none())); + assert!(possible_sleds + .iter() + .all(|sled| sled.affinity_policy.is_none())); + assert!(possible_sleds + .iter() + .all(|sled| sled.anti_affinity_policy.is_none())); // Concurrently create large instances on sleds 0, 2, 3. let groups = []; - let all_groups = AllGroups::create(&opctx, &datastore, &authz_project, &groups) - .await; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; let instances = [ - Instance::new() - .use_many_resources() - .sled(sleds[0].id()), - Instance::new() - .use_many_resources() - .sled(sleds[2].id()), - Instance::new() - .use_many_resources() - .sled(sleds[3].id()), - + Instance::new().use_many_resources().sled(sleds[0].id()), + Instance::new().use_many_resources().sled(sleds[2].id()), + Instance::new().use_many_resources().sled(sleds[3].id()), ]; for instance in instances { instance @@ -2727,19 +2753,28 @@ pub(in crate::db::datastore) mod test { // Inserting onto sleds[0, 2, 3] should fail - there shouldn't // be enough space on these sleds. for i in [0, 2, 3] { - assert!(!test_instance.insert_resource( - &datastore, - PropolisUuid::new_v4(), - SledUuid::from_untyped_uuid(sleds[i].id()), - ).await, "Shouldn't have been able to insert into sleds[i]"); + assert!( + !test_instance + .insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[i].id()), + ) + .await, + "Shouldn't have been able to insert into sleds[i]" + ); } // Inserting into sleds[1] should succeed - assert!(test_instance.insert_resource( - &datastore, - PropolisUuid::new_v4(), - SledUuid::from_untyped_uuid(sleds[1].id()), - ).await); + assert!( + test_instance + .insert_resource( + &datastore, + PropolisUuid::new_v4(), + SledUuid::from_untyped_uuid(sleds[1].id()), + ) + .await + ); db.terminate().await; logctx.cleanup_successful(); From d55e5403386d75d7d3f0fdb690adcc451cfdf567 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 13 Feb 2025 11:38:01 -0800 Subject: [PATCH 44/84] Patch benchmark --- nexus/db-queries/benches/harness/db_utils.rs | 15 +++ nexus/db-queries/benches/sled_reservation.rs | 128 +++++++++++-------- 2 files changed, 92 insertions(+), 51 deletions(-) diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index a714d462b2d..375ddf95cc5 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -110,6 +110,21 @@ fn small_resource_request() -> Resources { ) } +/// Given a `sled_count`, returns the number of times a call to +/// `create_reservation` should succeed. +/// +/// This can be used to validate parameters before running benchmarks. +pub fn max_resource_request_count(sled_count: usize) -> usize { + let threads_per_request: usize = + small_resource_request().hardware_threads.0.try_into().unwrap(); + let threads_per_sled: usize = sled_system_hardware_for_test() + .usable_hardware_threads + .try_into() + .unwrap(); + + threads_per_sled * sled_count / threads_per_request +} + pub async fn create_reservation( opctx: &OpContext, db: &DataStore, diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 4b1d825beab..a6aff0b4e1d 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -5,7 +5,7 @@ //! Benchmarks creating sled reservations use criterion::black_box; -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use omicron_test_utils::dev; @@ -18,6 +18,7 @@ mod harness; use harness::db_utils::create_reservation; use harness::db_utils::delete_reservation; +use harness::db_utils::max_resource_request_count; use harness::TestHarness; ///////////////////////////////////////////////////////////////// @@ -35,6 +36,7 @@ struct TestParams { const VMM_PARAMS: [usize; 3] = [1, 8, 16]; const TASK_PARAMS: [usize; 3] = [1, 4, 8]; +const SLED_PARAMS: [usize; 3] = [1, 4, 8]; ///////////////////////////////////////////////////////////////// // @@ -45,6 +47,9 @@ const TASK_PARAMS: [usize; 3] = [1, 4, 8]; // ```bash // cargo bench -p nexus-db-queries // ``` +// +// You can also set the "SHOW_CONTENTION" environment variable to display +// additional data from CockroachDB tables about contention statistics. // Average a duration over a divisor. // @@ -66,16 +71,13 @@ async fn reserve_vmms_and_return_average_duration( params: &TestParams, opctx: &OpContext, db: &DataStore, + cleanup_barrier: &Barrier, ) -> Duration { let mut vmm_ids = Vec::with_capacity(params.vmms); - let start = Instant::now(); - - // Clippy: We don't want to move this block outside of "black_box", even though it - // isn't returning anything. That would defeat the whole point of using "black_box", - // which is to avoid profiling code that is optimized based on the surrounding - // benchmark function. - #[allow(clippy::unit_arg)] - black_box({ + + let duration = black_box({ + let start = Instant::now(); + // Create all the requested vmms. // // Note that all prior reservations will remain in the DB as we continue @@ -87,19 +89,22 @@ async fn reserve_vmms_and_return_average_duration( .expect("Failed to provision vmm"), ); } + // Return the "average time to provision a single vmm". + // + // This normalizes the results, regardless of how many vmms we are provisioning. + // + // Note that we expect additional contention to create more work, but it's difficult to + // normalize "how much work is being created by contention". + average_duration(start.elapsed(), params.vmms) }); - // Return the "average time to provision a single vmm". - // - // This normalizes the results, regardless of how many vmms we are provisioning. - // - // Note that we expect additional contention to create more work, but it's difficult to - // normalize "how much work is being created by contention". - let duration = average_duration(start.elapsed(), params.vmms); - // Clean up all our vmms. // // We don't really care how long this takes, so we omit it from the tracking time. + // + // Use a barrier to ensure this does not interfere with the work of + // concurrent reservations. + cleanup_barrier.wait().await; for vmm_id in vmm_ids.drain(..) { delete_reservation(opctx, db, vmm_id) .await @@ -141,7 +146,7 @@ async fn bench_reservation( // ... and then actually do the benchmark reserve_vmms_and_return_average_duration( - ¶ms, &opctx, &db, + ¶ms, &opctx, &db, &barrier, ) .await } @@ -194,39 +199,58 @@ fn sled_reservation_benchmark(c: &mut Criterion) { rt.block_on(harness::setup_db(&logctx.log)); let mut group = c.benchmark_group("vmm-reservation"); - for tasks in TASK_PARAMS { - for vmms in VMM_PARAMS { - let params = TestParams { vmms, tasks }; - let name = format!("{tasks}-tasks-{vmms}-vmms"); - - // Initialize the harness before calling "bench_function" so - // that the "warm-up" calls to "bench_function" are actually useful - // at warming up the database. - // - // This mitigates any database-caching issues like "loading schema - // on boot", or "connection pooling", as the pool stays the same - // between calls to the benchmark function. - let log = logctx.log.clone(); - let harness = rt.block_on(async move { - const SLED_COUNT: usize = 4; - TestHarness::new(&log, SLED_COUNT).await - }); - - // Actually invoke the benchmark. - group.bench_function(&name, |b| { - b.to_async(&rt).iter_custom(|iters| { - let opctx = harness.opctx(); - let db = harness.db(); - async move { bench_reservation(opctx, db, params, iters).await } - }) - }); - - // Clean-up the harness; we'll use a new database between - // varations in parameters. - rt.block_on(async move { - harness.print_contention().await; - harness.terminate().await; - }); + + // Flat sampling is recommended for benchmarks which run "longer than + // milliseconds", which is the case for these benches. + // + // See: https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html + group.sampling_mode(SamplingMode::Flat); + + for sleds in SLED_PARAMS { + for tasks in TASK_PARAMS { + for vmms in VMM_PARAMS { + let params = TestParams { vmms, tasks }; + let name = format!("{sleds}-sleds-{tasks}-tasks-{vmms}-vmms"); + + if tasks * vmms > max_resource_request_count(sleds) { + eprintln!( + "{name} would request too many VMMs; skipping..." + ); + continue; + } + + // Initialize the harness before calling "bench_function" so + // that the "warm-up" calls to "bench_function" are actually useful + // at warming up the database. + // + // This mitigates any database-caching issues like "loading schema + // on boot", or "connection pooling", as the pool stays the same + // between calls to the benchmark function. + let log = logctx.log.clone(); + let harness = + rt.block_on( + async move { TestHarness::new(&log, sleds).await }, + ); + + // Actually invoke the benchmark. + group.bench_function(&name, |b| { + b.to_async(&rt).iter_custom(|iters| { + let opctx = harness.opctx(); + let db = harness.db(); + async move { + bench_reservation(opctx, db, params, iters).await + } + }) + }); + // Clean-up the harness; we'll use a new database between + // varations in parameters. + rt.block_on(async move { + if std::env::var("SHOW_CONTENTION").is_ok() { + harness.print_contention().await; + } + harness.terminate().await; + }); + } } } group.finish(); @@ -241,6 +265,8 @@ criterion_group!( // - Higher noise threshold, to avoid avoid false positive change detection config = Criterion::default() .sample_size(10) + // Allow for 10% variance in performance without identifying performance + // improvements/regressions .noise_threshold(0.10); targets = sled_reservation_benchmark ); From f7106879a5b9ddb4e7140271ad07a38683f7bcdf Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Feb 2025 10:36:59 -0800 Subject: [PATCH 45/84] Partway through affinity group testing --- common/src/api/external/mod.rs | 4 +- nexus/db-queries/benches/harness/db_utils.rs | 175 ++++++- nexus/db-queries/benches/harness/mod.rs | 10 +- nexus/db-queries/benches/sled_reservation.rs | 456 ++++++++++++++----- 4 files changed, 519 insertions(+), 126 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f4b657a1a59..3e9f71f7f10 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1329,7 +1329,9 @@ pub enum InstanceAutoRestartPolicy { /// Affinity policy used to describe "what to do when a request cannot be satisfied" /// /// Used for both Affinity and Anti-Affinity Groups -#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[derive( + Clone, Copy, Debug, Deserialize, Hash, Eq, Serialize, PartialEq, JsonSchema, +)] #[serde(rename_all = "snake_case")] pub enum AffinityPolicy { /// If the affinity request cannot be satisfied, allow it anyway. diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index 375ddf95cc5..a116743da9a 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -14,8 +14,12 @@ use anyhow::Context; use anyhow::Result; +use chrono::Utc; use nexus_db_model::ByteCount; use nexus_db_model::Generation; +use nexus_db_model::Instance; +use nexus_db_model::InstanceRuntimeState; +use nexus_db_model::InstanceState; use nexus_db_model::Project; use nexus_db_model::Resources; use nexus_db_model::Sled; @@ -25,13 +29,17 @@ use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::DataStore; use nexus_types::external_api::params; +use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use std::net::Ipv6Addr; use std::net::SocketAddrV6; +use std::str::FromStr; use uuid::Uuid; pub async fn create_project( @@ -125,11 +133,176 @@ pub fn max_resource_request_count(sled_count: usize) -> usize { threads_per_sled * sled_count / threads_per_request } +// Helper function for creating an instance without a VMM. +pub async fn create_instance_record( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, +) -> InstanceUuid { + let instance = Instance::new( + InstanceUuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + ncpus: 2i64.try_into().unwrap(), + memory: external::ByteCount::from_gibibytes_u32(16).into(), + hostname: "myhostname".try_into().unwrap(), + user_data: Vec::new(), + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: Vec::new(), + disks: Vec::new(), + boot_disk: None, + ssh_public_keys: None, + start: false, + auto_restart_policy: Default::default(), + }, + ); + + let instance = datastore + .project_create_instance(&opctx, &authz_project, instance) + .await + .unwrap(); + + let id = InstanceUuid::from_untyped_uuid(instance.id()); + datastore + .instance_update_runtime( + &id, + &InstanceRuntimeState { + nexus_state: InstanceState::NoVmm, + time_updated: Utc::now(), + propolis_id: None, + migration_id: None, + dst_propolis_id: None, + gen: Generation::from(Generation::new().0.next()), + time_last_auto_restarted: None, + }, + ) + .await + .expect("Failed to update runtime state"); + + id +} +pub async fn delete_instance_record( + opctx: &OpContext, + datastore: &DataStore, + instance_id: InstanceUuid, +) { + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance_id.into_untyped_uuid()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.project_delete_instance(&opctx, &authz_instance).await.unwrap(); +} + +pub async fn create_affinity_group( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + group_name: &'static str, + policy: external::AffinityPolicy, +) { + db.affinity_group_create( + &opctx, + &authz_project, + nexus_db_model::AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap(); +} + +pub async fn create_anti_affinity_group( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + group_name: &'static str, + policy: external::AffinityPolicy, +) { + db.anti_affinity_group_create( + &opctx, + &authz_project, + nexus_db_model::AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap(); +} + +pub async fn create_affinity_group_member( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, + instance_id: InstanceUuid, +) -> Result<()> { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Modify) + .await?; + + db.affinity_group_member_add( + opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance_id), + ) + .await?; + Ok(()) +} + +pub async fn create_anti_affinity_group_member( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, + instance_id: InstanceUuid, +) -> Result<()> { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .anti_affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Modify) + .await?; + + db.anti_affinity_group_member_add( + opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance_id), + ) + .await?; + Ok(()) +} + pub async fn create_reservation( opctx: &OpContext, db: &DataStore, + instance_id: InstanceUuid, ) -> Result { - let instance_id = InstanceUuid::new_v4(); let vmm_id = PropolisUuid::new_v4(); loop { diff --git a/nexus/db-queries/benches/harness/mod.rs b/nexus/db-queries/benches/harness/mod.rs index 1b244e083ae..e4d8b1baf3a 100644 --- a/nexus/db-queries/benches/harness/mod.rs +++ b/nexus/db-queries/benches/harness/mod.rs @@ -7,6 +7,7 @@ //! This structure shares logic between benchmarks, making it easy //! to perform shared tasks such as creating contention for reservations. +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DataStore; @@ -23,6 +24,7 @@ use db_utils::*; pub struct TestHarness { db: TestDatabase, + authz_project: authz::Project, } struct ContentionQuery { @@ -83,11 +85,11 @@ impl TestHarness { pub async fn new(log: &Logger, sled_count: usize) -> Self { let db = TestDatabase::new_with_datastore(log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let (_authz_project, _project) = + let (authz_project, _project) = create_project(&opctx, &datastore).await; create_sleds(&datastore, sled_count).await; - Self { db } + Self { db, authz_project } } /// Emit internal CockroachDb information about contention @@ -159,6 +161,10 @@ impl TestHarness { client.cleanup().await.expect("Failed to clean up db connection"); } + pub fn authz_project(&self) -> authz::Project { + self.authz_project.clone() + } + /// Returns an owned reference to OpContext pub fn opctx(&self) -> Arc { Arc::new(self.db.opctx().child(std::collections::BTreeMap::new())) diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index a6aff0b4e1d..1f8483dbe29 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -6,9 +6,13 @@ use criterion::black_box; use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; +use omicron_common::api::external; use omicron_test_utils::dev; +use omicron_uuid_kinds::InstanceUuid; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -16,7 +20,13 @@ use tokio::sync::Barrier; mod harness; +use harness::db_utils::create_affinity_group; +use harness::db_utils::create_affinity_group_member; +use harness::db_utils::create_anti_affinity_group; +use harness::db_utils::create_anti_affinity_group_member; +use harness::db_utils::create_instance_record; use harness::db_utils::create_reservation; +use harness::db_utils::delete_instance_record; use harness::db_utils::delete_reservation; use harness::db_utils::max_resource_request_count; use harness::TestHarness; @@ -27,11 +37,45 @@ use harness::TestHarness; // // Describes varations between otherwise shared test logic -#[derive(Copy, Clone)] +#[derive(Clone, Hash, Eq, PartialEq)] +enum GroupType { + Affinity, + AntiAffinity, +} + +#[derive(Clone, Hash, Eq, PartialEq)] +struct GroupInfo { + // Name of affinity/anti-affinity group + name: &'static str, + // Policy of the group + policy: external::AffinityPolicy, + // Type of group + flavor: GroupType, +} + +#[derive(Clone)] +struct InstanceGroups { + // An instance should belong to all these groups. + belongs_to: Vec, +} + +#[derive(Clone)] +struct InstanceGroupPattern { + description: &'static str, + // These "instance group settings" should be applied. + // + // NOTE: Currently, we rotate through these groups + group_pattern: Vec, +} + +#[derive(Clone)] struct TestParams { // Number of vmms to provision from the task-under-test vmms: usize, + // Number of tasks to concurrent provision vmms tasks: usize, + // The pattern of allocations + group_pattern: InstanceGroupPattern, } const VMM_PARAMS: [usize; 3] = [1, 8, 16]; @@ -64,6 +108,72 @@ fn average_duration(duration: Duration, divisor: usize) -> Duration { ) } +async fn create_instances_with_groups( + params: &TestParams, + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + task_num: usize, +) -> Vec { + let mut instance_ids = Vec::with_capacity(params.vmms); + for i in 0..params.vmms { + let instance_id = create_instance_record( + opctx, + db, + authz_project, + &format!("task-{task_num}-instance-{i}"), + ) + .await; + + instance_ids.push(instance_id); + + let patterns = params.group_pattern.group_pattern.len(); + if patterns > 0 { + let groups = + ¶ms.group_pattern.group_pattern[i % patterns].belongs_to; + for group in groups { + match group.flavor { + GroupType::Affinity => { + create_affinity_group_member( + opctx, + db, + group.name, + instance_id, + ) + .await + .expect("Failed to add instance to affinity group"); + } + GroupType::AntiAffinity => { + create_anti_affinity_group_member( + opctx, + db, + group.name, + instance_id, + ) + .await + .expect( + "Failed to add instance to anti-affinity group", + ); + } + } + } + } + } + instance_ids +} + +async fn destroy_instances_with_groups( + opctx: &OpContext, + db: &DataStore, + instances: Vec, +) { + for instance_id in instances { + delete_instance_record(opctx, db, instance_id).await; + } + + // TODO: Delete groups too +} + // Reserves "params.vmms" vmms, and later deletes their reservations. // // Returns the average time to provision a single vmm. @@ -71,10 +181,22 @@ async fn reserve_vmms_and_return_average_duration( params: &TestParams, opctx: &OpContext, db: &DataStore, - cleanup_barrier: &Barrier, + authz_project: &authz::Project, + task_num: usize, + barrier: &Barrier, ) -> Duration { + let instance_ids = create_instances_with_groups( + params, + opctx, + db, + authz_project, + task_num, + ) + .await; let mut vmm_ids = Vec::with_capacity(params.vmms); + // Wait for all tasks to start at roughly the same time + barrier.wait().await; let duration = black_box({ let start = Instant::now(); @@ -83,8 +205,9 @@ async fn reserve_vmms_and_return_average_duration( // Note that all prior reservations will remain in the DB as we continue // provisioning the "next" vmm. for _ in 0..params.vmms { + let instance_id = InstanceUuid::new_v4(); vmm_ids.push( - create_reservation(opctx, db) + create_reservation(opctx, db, instance_id) .await .expect("Failed to provision vmm"), ); @@ -104,92 +227,134 @@ async fn reserve_vmms_and_return_average_duration( // // Use a barrier to ensure this does not interfere with the work of // concurrent reservations. - cleanup_barrier.wait().await; + barrier.wait().await; for vmm_id in vmm_ids.drain(..) { delete_reservation(opctx, db, vmm_id) .await .expect("Failed to delete vmm"); } + // Additionally, destroy all our instance records (and their affinity group memberships) + destroy_instances_with_groups(opctx, db, instance_ids).await; duration } +async fn create_test_groups( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + params: &TestParams, +) { + let all_groups: HashSet<_> = params + .group_pattern + .group_pattern + .iter() + .map(|groups| groups.belongs_to.iter()) + .flatten() + .collect(); + for group in all_groups { + match group.flavor { + GroupType::Affinity => { + create_affinity_group( + opctx, + db, + authz_project, + group.name, + group.policy, + ) + .await; + } + GroupType::AntiAffinity => { + create_anti_affinity_group( + opctx, + db, + authz_project, + group.name, + group.policy, + ) + .await; + } + } + } +} + async fn bench_reservation( opctx: Arc, db: Arc, + authz_project: authz::Project, params: TestParams, iterations: u64, ) -> Duration { - let duration = { - let mut total_duration = Duration::ZERO; - - // Each iteration is an "attempt" at the test. - for _ in 0..iterations { - // Within each attempt, we spawn the tasks requested. - let mut set = tokio::task::JoinSet::new(); - - // This barrier exists to lessen the impact of "task spawning" on the benchmark. - // - // We want to have all tasks run as concurrently as possible, since we're trying to - // measure contention explicitly. - let barrier = Arc::new(Barrier::new(params.tasks)); - - for _ in 0..params.tasks { - set.spawn({ - let opctx = opctx.clone(); - let db = db.clone(); - let barrier = barrier.clone(); - - async move { - // Wait until all tasks are ready... - barrier.wait().await; - - // ... and then actually do the benchmark - reserve_vmms_and_return_average_duration( - ¶ms, &opctx, &db, &barrier, - ) - .await - } - }); - } + let mut total_duration = Duration::ZERO; + + create_test_groups(&opctx, &db, &authz_project, ¶ms).await; - // The sum of "average time to provision a single vmm" across all tasks. - let all_tasks_duration = set - .join_all() - .await - .into_iter() - .fold(Duration::ZERO, |acc, x| acc + x); - - // The normalized "time to provision a single vmm", across both: - // - The number of vmms reserved by each task, and - // - The number of tasks - // - // As an example, if we provision 10 vmms, and have 5 tasks, and we assume - // that VM provisioning time is exactly one second (contention has no impact, somehow): - // - // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average - // duration of "1 second". - // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds - // (1 second average * 5 tasks). - // - So, we'd increment our "total_duration" by "1 second per vmm", which has been - // normalized cross both the tasks and quantities of vmms. - // - // Why bother doing this? - // - // When we perform this normalization, we can vary the "total vmms provisioned" as well - // as "total tasks" significantly, but easily compare test durations with one another. - // - // For example: if the total number of vmms has no impact on the next provisioning - // request, we should see similar durations for "100 vmms reserved" vs "1 vmm - // reserved". However, if more vmms actually make reservation times slower, we'll see - // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: - total_duration += - average_duration(all_tasks_duration, params.tasks); + // Each iteration is an "attempt" at the test. + for _ in 0..iterations { + // Within each attempt, we spawn the tasks requested. + let mut set = tokio::task::JoinSet::new(); + + // This barrier exists to lessen the impact of "task spawning" on the benchmark. + // + // We want to have all tasks run as concurrently as possible, since we're trying to + // measure contention explicitly. + let barrier = Arc::new(Barrier::new(params.tasks)); + + for task_num in 0..params.tasks { + set.spawn({ + let opctx = opctx.clone(); + let db = db.clone(); + let authz_project = authz_project.clone(); + let barrier = barrier.clone(); + let params = params.clone(); + + async move { + reserve_vmms_and_return_average_duration( + ¶ms, + &opctx, + &db, + &authz_project, + task_num, + &barrier, + ) + .await + } + }); } - total_duration - }; - duration + // The sum of "average time to provision a single vmm" across all tasks. + let all_tasks_duration = set + .join_all() + .await + .into_iter() + .fold(Duration::ZERO, |acc, x| acc + x); + + // The normalized "time to provision a single vmm", across both: + // - The number of vmms reserved by each task, and + // - The number of tasks + // + // As an example, if we provision 10 vmms, and have 5 tasks, and we assume + // that VM provisioning time is exactly one second (contention has no impact, somehow): + // + // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average + // duration of "1 second". + // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds + // (1 second average * 5 tasks). + // - So, we'd increment our "total_duration" by "1 second per vmm", which has been + // normalized cross both the tasks and quantities of vmms. + // + // Why bother doing this? + // + // When we perform this normalization, we can vary the "total vmms provisioned" as well + // as "total tasks" significantly, but easily compare test durations with one another. + // + // For example: if the total number of vmms has no impact on the next provisioning + // request, we should see similar durations for "100 vmms reserved" vs "1 vmm + // reserved". However, if more vmms actually make reservation times slower, we'll see + // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: + total_duration += average_duration(all_tasks_duration, params.tasks); + } + total_duration } fn sled_reservation_benchmark(c: &mut Criterion) { @@ -198,62 +363,109 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(harness::setup_db(&logctx.log)); - let mut group = c.benchmark_group("vmm-reservation"); - - // Flat sampling is recommended for benchmarks which run "longer than - // milliseconds", which is the case for these benches. - // - // See: https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html - group.sampling_mode(SamplingMode::Flat); - - for sleds in SLED_PARAMS { - for tasks in TASK_PARAMS { - for vmms in VMM_PARAMS { - let params = TestParams { vmms, tasks }; - let name = format!("{sleds}-sleds-{tasks}-tasks-{vmms}-vmms"); - - if tasks * vmms > max_resource_request_count(sleds) { - eprintln!( - "{name} would request too many VMMs; skipping..." - ); - continue; - } + let group_patterns = [ + // No Affinity Groups + InstanceGroupPattern { + description: "no-groups", + group_pattern: vec![], + }, + // Alternating "Affinity Group" and "Anti-Affinity Group", both permissive + InstanceGroupPattern { + description: "affinity-groups-permissive", + group_pattern: vec![ + InstanceGroups { + belongs_to: vec![GroupInfo { + name: "affinity-1", + policy: external::AffinityPolicy::Allow, + flavor: GroupType::Affinity, + }], + }, + InstanceGroups { + belongs_to: vec![GroupInfo { + name: "anti-affinity-1", + policy: external::AffinityPolicy::Allow, + flavor: GroupType::AntiAffinity, + }], + }, + ], + }, + // TODO create a test for "policy = Fail" groups? + ]; + + for grouping in &group_patterns { + let mut group = c.benchmark_group(format!( + "vmm-reservation-{}", + grouping.description + )); + + // Flat sampling is recommended for benchmarks which run "longer than + // milliseconds", which is the case for these benches. + // + // See: https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html + group.sampling_mode(SamplingMode::Flat); + + for sleds in SLED_PARAMS { + for tasks in TASK_PARAMS { + for vmms in VMM_PARAMS { + let params = TestParams { + vmms, + tasks, + group_pattern: grouping.clone(), + }; + let name = + format!("{sleds}-sleds-{tasks}-tasks-{vmms}-vmms"); + + if tasks * vmms > max_resource_request_count(sleds) { + eprintln!( + "{name} would request too many VMMs; skipping..." + ); + continue; + } - // Initialize the harness before calling "bench_function" so - // that the "warm-up" calls to "bench_function" are actually useful - // at warming up the database. - // - // This mitigates any database-caching issues like "loading schema - // on boot", or "connection pooling", as the pool stays the same - // between calls to the benchmark function. - let log = logctx.log.clone(); - let harness = - rt.block_on( - async move { TestHarness::new(&log, sleds).await }, - ); - - // Actually invoke the benchmark. - group.bench_function(&name, |b| { - b.to_async(&rt).iter_custom(|iters| { - let opctx = harness.opctx(); - let db = harness.db(); - async move { - bench_reservation(opctx, db, params, iters).await + // Initialize the harness before calling "bench_function" so + // that the "warm-up" calls to "bench_function" are actually useful + // at warming up the database. + // + // This mitigates any database-caching issues like "loading schema + // on boot", or "connection pooling", as the pool stays the same + // between calls to the benchmark function. + let log = logctx.log.clone(); + let harness = rt.block_on(async move { + TestHarness::new(&log, sleds).await + }); + + // Actually invoke the benchmark. + group.bench_function(&name, |b| { + b.to_async(&rt).iter_custom(|iters| { + let opctx = harness.opctx(); + let db = harness.db(); + let authz_project = harness.authz_project(); + let params = params.clone(); + async move { + bench_reservation( + opctx, + db, + authz_project, + params, + iters, + ) + .await + } + }) + }); + // Clean-up the harness; we'll use a new database between + // varations in parameters. + rt.block_on(async move { + if std::env::var("SHOW_CONTENTION").is_ok() { + harness.print_contention().await; } - }) - }); - // Clean-up the harness; we'll use a new database between - // varations in parameters. - rt.block_on(async move { - if std::env::var("SHOW_CONTENTION").is_ok() { - harness.print_contention().await; - } - harness.terminate().await; - }); + harness.terminate().await; + }); + } } } + group.finish(); } - group.finish(); logctx.cleanup_successful(); } From 82b028cafd5b474b7b7bb2f84a763d3b036c4ffc Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Feb 2025 10:55:58 -0800 Subject: [PATCH 46/84] cache instance/group records --- nexus/db-queries/benches/harness/db_utils.rs | 42 ++++++++++ nexus/db-queries/benches/sled_reservation.rs | 82 ++++++++++++++++---- 2 files changed, 108 insertions(+), 16 deletions(-) diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index a116743da9a..33fb48c86f7 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -226,6 +226,27 @@ pub async fn create_affinity_group( .unwrap(); } +pub async fn delete_affinity_group( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, +) { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Delete) + .await.unwrap(); + + db.affinity_group_delete( + &opctx, + &authz_group, + ) + .await + .unwrap(); +} + pub async fn create_anti_affinity_group( opctx: &OpContext, db: &DataStore, @@ -252,6 +273,27 @@ pub async fn create_anti_affinity_group( .unwrap(); } +pub async fn delete_anti_affinity_group( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, +) { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .anti_affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Delete) + .await.unwrap(); + + db.anti_affinity_group_delete( + &opctx, + &authz_group, + ) + .await + .unwrap(); +} + pub async fn create_affinity_group_member( opctx: &OpContext, db: &DataStore, diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 1f8483dbe29..f463f92c612 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -26,6 +26,8 @@ use harness::db_utils::create_anti_affinity_group; use harness::db_utils::create_anti_affinity_group_member; use harness::db_utils::create_instance_record; use harness::db_utils::create_reservation; +use harness::db_utils::delete_affinity_group; +use harness::db_utils::delete_anti_affinity_group; use harness::db_utils::delete_instance_record; use harness::db_utils::delete_reservation; use harness::db_utils::max_resource_request_count; @@ -181,18 +183,10 @@ async fn reserve_vmms_and_return_average_duration( params: &TestParams, opctx: &OpContext, db: &DataStore, - authz_project: &authz::Project, - task_num: usize, + instance_ids: &Vec, barrier: &Barrier, ) -> Duration { - let instance_ids = create_instances_with_groups( - params, - opctx, - db, - authz_project, - task_num, - ) - .await; + assert_eq!(instance_ids.len(), params.vmms, "Not enough instances to provision"); let mut vmm_ids = Vec::with_capacity(params.vmms); // Wait for all tasks to start at roughly the same time @@ -233,9 +227,6 @@ async fn reserve_vmms_and_return_average_duration( .await .expect("Failed to delete vmm"); } - // Additionally, destroy all our instance records (and their affinity group memberships) - destroy_instances_with_groups(opctx, db, instance_ids).await; - duration } @@ -278,6 +269,40 @@ async fn create_test_groups( } } +async fn delete_test_groups( + opctx: &OpContext, + db: &DataStore, + params: &TestParams, +) { + let all_groups: HashSet<_> = params + .group_pattern + .group_pattern + .iter() + .map(|groups| groups.belongs_to.iter()) + .flatten() + .collect(); + for group in all_groups { + match group.flavor { + GroupType::Affinity => { + delete_affinity_group( + opctx, + db, + group.name, + ) + .await; + } + GroupType::AntiAffinity => { + delete_anti_affinity_group( + opctx, + db, + group.name, + ) + .await; + } + } + } +} + async fn bench_reservation( opctx: Arc, db: Arc, @@ -287,7 +312,24 @@ async fn bench_reservation( ) -> Duration { let mut total_duration = Duration::ZERO; + // TODO: Can we avoid passing "authz_project" everywhere, and just do + // lookups in db_utils? + + // Create all groups and instances belonging to those groups before + // we actually do any iterations. This is a slight optimization to + // make the benchmarks - focused on VMM reservation time - a little faster. create_test_groups(&opctx, &db, &authz_project, ¶ms).await; + let mut task_instances = vec![]; + for task_num in 0..params.tasks { + task_instances.push(create_instances_with_groups( + ¶ms, + &opctx, + &db, + &authz_project, + task_num, + ) + .await); + } // Each iteration is an "attempt" at the test. for _ in 0..iterations { @@ -304,17 +346,16 @@ async fn bench_reservation( set.spawn({ let opctx = opctx.clone(); let db = db.clone(); - let authz_project = authz_project.clone(); let barrier = barrier.clone(); let params = params.clone(); + let instances = task_instances[task_num].clone(); async move { reserve_vmms_and_return_average_duration( ¶ms, &opctx, &db, - &authz_project, - task_num, + &instances, &barrier, ) .await @@ -354,6 +395,12 @@ async fn bench_reservation( // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: total_duration += average_duration(all_tasks_duration, params.tasks); } + // Destroy all our instance records (and their affinity group memberships) + for instance_ids in task_instances { + destroy_instances_with_groups(&opctx, &db, instance_ids).await; + } + + delete_test_groups(&opctx, &db, ¶ms).await; total_duration } @@ -415,6 +462,9 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let name = format!("{sleds}-sleds-{tasks}-tasks-{vmms}-vmms"); + // TODO: This can also fail depending on the group requirements, + // if we're using policy = fail. Maybe construct "TestParams" + // differently to let each test decide this. if tasks * vmms > max_resource_request_count(sleds) { eprintln!( "{name} would request too many VMMs; skipping..." From 43f4437be9f5307ebee1d8c1f13f913b18680120 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Feb 2025 10:57:40 -0800 Subject: [PATCH 47/84] benchmark updates --- common/src/api/external/mod.rs | 4 +- nexus/db-queries/benches/harness/db_utils.rs | 232 +++++++- nexus/db-queries/benches/harness/mod.rs | 10 +- nexus/db-queries/benches/sled_reservation.rs | 528 ++++++++++++++----- 4 files changed, 650 insertions(+), 124 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f4b657a1a59..3e9f71f7f10 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1329,7 +1329,9 @@ pub enum InstanceAutoRestartPolicy { /// Affinity policy used to describe "what to do when a request cannot be satisfied" /// /// Used for both Affinity and Anti-Affinity Groups -#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[derive( + Clone, Copy, Debug, Deserialize, Hash, Eq, Serialize, PartialEq, JsonSchema, +)] #[serde(rename_all = "snake_case")] pub enum AffinityPolicy { /// If the affinity request cannot be satisfied, allow it anyway. diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index a714d462b2d..33fb48c86f7 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -14,8 +14,12 @@ use anyhow::Context; use anyhow::Result; +use chrono::Utc; use nexus_db_model::ByteCount; use nexus_db_model::Generation; +use nexus_db_model::Instance; +use nexus_db_model::InstanceRuntimeState; +use nexus_db_model::InstanceState; use nexus_db_model::Project; use nexus_db_model::Resources; use nexus_db_model::Sled; @@ -25,13 +29,17 @@ use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::DataStore; use nexus_types::external_api::params; +use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use std::net::Ipv6Addr; use std::net::SocketAddrV6; +use std::str::FromStr; use uuid::Uuid; pub async fn create_project( @@ -110,11 +118,233 @@ fn small_resource_request() -> Resources { ) } +/// Given a `sled_count`, returns the number of times a call to +/// `create_reservation` should succeed. +/// +/// This can be used to validate parameters before running benchmarks. +pub fn max_resource_request_count(sled_count: usize) -> usize { + let threads_per_request: usize = + small_resource_request().hardware_threads.0.try_into().unwrap(); + let threads_per_sled: usize = sled_system_hardware_for_test() + .usable_hardware_threads + .try_into() + .unwrap(); + + threads_per_sled * sled_count / threads_per_request +} + +// Helper function for creating an instance without a VMM. +pub async fn create_instance_record( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, +) -> InstanceUuid { + let instance = Instance::new( + InstanceUuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + ncpus: 2i64.try_into().unwrap(), + memory: external::ByteCount::from_gibibytes_u32(16).into(), + hostname: "myhostname".try_into().unwrap(), + user_data: Vec::new(), + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: Vec::new(), + disks: Vec::new(), + boot_disk: None, + ssh_public_keys: None, + start: false, + auto_restart_policy: Default::default(), + }, + ); + + let instance = datastore + .project_create_instance(&opctx, &authz_project, instance) + .await + .unwrap(); + + let id = InstanceUuid::from_untyped_uuid(instance.id()); + datastore + .instance_update_runtime( + &id, + &InstanceRuntimeState { + nexus_state: InstanceState::NoVmm, + time_updated: Utc::now(), + propolis_id: None, + migration_id: None, + dst_propolis_id: None, + gen: Generation::from(Generation::new().0.next()), + time_last_auto_restarted: None, + }, + ) + .await + .expect("Failed to update runtime state"); + + id +} +pub async fn delete_instance_record( + opctx: &OpContext, + datastore: &DataStore, + instance_id: InstanceUuid, +) { + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance_id.into_untyped_uuid()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.project_delete_instance(&opctx, &authz_instance).await.unwrap(); +} + +pub async fn create_affinity_group( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + group_name: &'static str, + policy: external::AffinityPolicy, +) { + db.affinity_group_create( + &opctx, + &authz_project, + nexus_db_model::AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap(); +} + +pub async fn delete_affinity_group( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, +) { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Delete) + .await.unwrap(); + + db.affinity_group_delete( + &opctx, + &authz_group, + ) + .await + .unwrap(); +} + +pub async fn create_anti_affinity_group( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + group_name: &'static str, + policy: external::AffinityPolicy, +) { + db.anti_affinity_group_create( + &opctx, + &authz_project, + nexus_db_model::AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap(); +} + +pub async fn delete_anti_affinity_group( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, +) { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .anti_affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Delete) + .await.unwrap(); + + db.anti_affinity_group_delete( + &opctx, + &authz_group, + ) + .await + .unwrap(); +} + +pub async fn create_affinity_group_member( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, + instance_id: InstanceUuid, +) -> Result<()> { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Modify) + .await?; + + db.affinity_group_member_add( + opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance_id), + ) + .await?; + Ok(()) +} + +pub async fn create_anti_affinity_group_member( + opctx: &OpContext, + db: &DataStore, + group_name: &'static str, + instance_id: InstanceUuid, +) -> Result<()> { + let project = external::Name::from_str("project").unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .anti_affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Modify) + .await?; + + db.anti_affinity_group_member_add( + opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance_id), + ) + .await?; + Ok(()) +} + pub async fn create_reservation( opctx: &OpContext, db: &DataStore, + instance_id: InstanceUuid, ) -> Result { - let instance_id = InstanceUuid::new_v4(); let vmm_id = PropolisUuid::new_v4(); loop { diff --git a/nexus/db-queries/benches/harness/mod.rs b/nexus/db-queries/benches/harness/mod.rs index 1b244e083ae..e4d8b1baf3a 100644 --- a/nexus/db-queries/benches/harness/mod.rs +++ b/nexus/db-queries/benches/harness/mod.rs @@ -7,6 +7,7 @@ //! This structure shares logic between benchmarks, making it easy //! to perform shared tasks such as creating contention for reservations. +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DataStore; @@ -23,6 +24,7 @@ use db_utils::*; pub struct TestHarness { db: TestDatabase, + authz_project: authz::Project, } struct ContentionQuery { @@ -83,11 +85,11 @@ impl TestHarness { pub async fn new(log: &Logger, sled_count: usize) -> Self { let db = TestDatabase::new_with_datastore(log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let (_authz_project, _project) = + let (authz_project, _project) = create_project(&opctx, &datastore).await; create_sleds(&datastore, sled_count).await; - Self { db } + Self { db, authz_project } } /// Emit internal CockroachDb information about contention @@ -159,6 +161,10 @@ impl TestHarness { client.cleanup().await.expect("Failed to clean up db connection"); } + pub fn authz_project(&self) -> authz::Project { + self.authz_project.clone() + } + /// Returns an owned reference to OpContext pub fn opctx(&self) -> Arc { Arc::new(self.db.opctx().child(std::collections::BTreeMap::new())) diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 4b1d825beab..f463f92c612 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -5,10 +5,14 @@ //! Benchmarks creating sled reservations use criterion::black_box; -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; +use omicron_common::api::external; use omicron_test_utils::dev; +use omicron_uuid_kinds::InstanceUuid; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -16,8 +20,17 @@ use tokio::sync::Barrier; mod harness; +use harness::db_utils::create_affinity_group; +use harness::db_utils::create_affinity_group_member; +use harness::db_utils::create_anti_affinity_group; +use harness::db_utils::create_anti_affinity_group_member; +use harness::db_utils::create_instance_record; use harness::db_utils::create_reservation; +use harness::db_utils::delete_affinity_group; +use harness::db_utils::delete_anti_affinity_group; +use harness::db_utils::delete_instance_record; use harness::db_utils::delete_reservation; +use harness::db_utils::max_resource_request_count; use harness::TestHarness; ///////////////////////////////////////////////////////////////// @@ -26,15 +39,50 @@ use harness::TestHarness; // // Describes varations between otherwise shared test logic -#[derive(Copy, Clone)] +#[derive(Clone, Hash, Eq, PartialEq)] +enum GroupType { + Affinity, + AntiAffinity, +} + +#[derive(Clone, Hash, Eq, PartialEq)] +struct GroupInfo { + // Name of affinity/anti-affinity group + name: &'static str, + // Policy of the group + policy: external::AffinityPolicy, + // Type of group + flavor: GroupType, +} + +#[derive(Clone)] +struct InstanceGroups { + // An instance should belong to all these groups. + belongs_to: Vec, +} + +#[derive(Clone)] +struct InstanceGroupPattern { + description: &'static str, + // These "instance group settings" should be applied. + // + // NOTE: Currently, we rotate through these groups + group_pattern: Vec, +} + +#[derive(Clone)] struct TestParams { // Number of vmms to provision from the task-under-test vmms: usize, + // Number of tasks to concurrent provision vmms tasks: usize, + // The pattern of allocations + group_pattern: InstanceGroupPattern, } const VMM_PARAMS: [usize; 3] = [1, 8, 16]; const TASK_PARAMS: [usize; 3] = [1, 4, 8]; +const SLED_PARAMS: [usize; 3] = [1, 4, 8]; ///////////////////////////////////////////////////////////////// // @@ -45,6 +93,9 @@ const TASK_PARAMS: [usize; 3] = [1, 4, 8]; // ```bash // cargo bench -p nexus-db-queries // ``` +// +// You can also set the "SHOW_CONTENTION" environment variable to display +// additional data from CockroachDB tables about contention statistics. // Average a duration over a divisor. // @@ -59,6 +110,72 @@ fn average_duration(duration: Duration, divisor: usize) -> Duration { ) } +async fn create_instances_with_groups( + params: &TestParams, + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + task_num: usize, +) -> Vec { + let mut instance_ids = Vec::with_capacity(params.vmms); + for i in 0..params.vmms { + let instance_id = create_instance_record( + opctx, + db, + authz_project, + &format!("task-{task_num}-instance-{i}"), + ) + .await; + + instance_ids.push(instance_id); + + let patterns = params.group_pattern.group_pattern.len(); + if patterns > 0 { + let groups = + ¶ms.group_pattern.group_pattern[i % patterns].belongs_to; + for group in groups { + match group.flavor { + GroupType::Affinity => { + create_affinity_group_member( + opctx, + db, + group.name, + instance_id, + ) + .await + .expect("Failed to add instance to affinity group"); + } + GroupType::AntiAffinity => { + create_anti_affinity_group_member( + opctx, + db, + group.name, + instance_id, + ) + .await + .expect( + "Failed to add instance to anti-affinity group", + ); + } + } + } + } + } + instance_ids +} + +async fn destroy_instances_with_groups( + opctx: &OpContext, + db: &DataStore, + instances: Vec, +) { + for instance_id in instances { + delete_instance_record(opctx, db, instance_id).await; + } + + // TODO: Delete groups too +} + // Reserves "params.vmms" vmms, and later deletes their reservations. // // Returns the average time to provision a single vmm. @@ -66,125 +183,225 @@ async fn reserve_vmms_and_return_average_duration( params: &TestParams, opctx: &OpContext, db: &DataStore, + instance_ids: &Vec, + barrier: &Barrier, ) -> Duration { + assert_eq!(instance_ids.len(), params.vmms, "Not enough instances to provision"); let mut vmm_ids = Vec::with_capacity(params.vmms); - let start = Instant::now(); - - // Clippy: We don't want to move this block outside of "black_box", even though it - // isn't returning anything. That would defeat the whole point of using "black_box", - // which is to avoid profiling code that is optimized based on the surrounding - // benchmark function. - #[allow(clippy::unit_arg)] - black_box({ + + // Wait for all tasks to start at roughly the same time + barrier.wait().await; + let duration = black_box({ + let start = Instant::now(); + // Create all the requested vmms. // // Note that all prior reservations will remain in the DB as we continue // provisioning the "next" vmm. for _ in 0..params.vmms { + let instance_id = InstanceUuid::new_v4(); vmm_ids.push( - create_reservation(opctx, db) + create_reservation(opctx, db, instance_id) .await .expect("Failed to provision vmm"), ); } + // Return the "average time to provision a single vmm". + // + // This normalizes the results, regardless of how many vmms we are provisioning. + // + // Note that we expect additional contention to create more work, but it's difficult to + // normalize "how much work is being created by contention". + average_duration(start.elapsed(), params.vmms) }); - // Return the "average time to provision a single vmm". - // - // This normalizes the results, regardless of how many vmms we are provisioning. - // - // Note that we expect additional contention to create more work, but it's difficult to - // normalize "how much work is being created by contention". - let duration = average_duration(start.elapsed(), params.vmms); - // Clean up all our vmms. // // We don't really care how long this takes, so we omit it from the tracking time. + // + // Use a barrier to ensure this does not interfere with the work of + // concurrent reservations. + barrier.wait().await; for vmm_id in vmm_ids.drain(..) { delete_reservation(opctx, db, vmm_id) .await .expect("Failed to delete vmm"); } - duration } +async fn create_test_groups( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + params: &TestParams, +) { + let all_groups: HashSet<_> = params + .group_pattern + .group_pattern + .iter() + .map(|groups| groups.belongs_to.iter()) + .flatten() + .collect(); + for group in all_groups { + match group.flavor { + GroupType::Affinity => { + create_affinity_group( + opctx, + db, + authz_project, + group.name, + group.policy, + ) + .await; + } + GroupType::AntiAffinity => { + create_anti_affinity_group( + opctx, + db, + authz_project, + group.name, + group.policy, + ) + .await; + } + } + } +} + +async fn delete_test_groups( + opctx: &OpContext, + db: &DataStore, + params: &TestParams, +) { + let all_groups: HashSet<_> = params + .group_pattern + .group_pattern + .iter() + .map(|groups| groups.belongs_to.iter()) + .flatten() + .collect(); + for group in all_groups { + match group.flavor { + GroupType::Affinity => { + delete_affinity_group( + opctx, + db, + group.name, + ) + .await; + } + GroupType::AntiAffinity => { + delete_anti_affinity_group( + opctx, + db, + group.name, + ) + .await; + } + } + } +} + async fn bench_reservation( opctx: Arc, db: Arc, + authz_project: authz::Project, params: TestParams, iterations: u64, ) -> Duration { - let duration = { - let mut total_duration = Duration::ZERO; - - // Each iteration is an "attempt" at the test. - for _ in 0..iterations { - // Within each attempt, we spawn the tasks requested. - let mut set = tokio::task::JoinSet::new(); - - // This barrier exists to lessen the impact of "task spawning" on the benchmark. - // - // We want to have all tasks run as concurrently as possible, since we're trying to - // measure contention explicitly. - let barrier = Arc::new(Barrier::new(params.tasks)); - - for _ in 0..params.tasks { - set.spawn({ - let opctx = opctx.clone(); - let db = db.clone(); - let barrier = barrier.clone(); - - async move { - // Wait until all tasks are ready... - barrier.wait().await; - - // ... and then actually do the benchmark - reserve_vmms_and_return_average_duration( - ¶ms, &opctx, &db, - ) - .await - } - }); - } + let mut total_duration = Duration::ZERO; + + // TODO: Can we avoid passing "authz_project" everywhere, and just do + // lookups in db_utils? + + // Create all groups and instances belonging to those groups before + // we actually do any iterations. This is a slight optimization to + // make the benchmarks - focused on VMM reservation time - a little faster. + create_test_groups(&opctx, &db, &authz_project, ¶ms).await; + let mut task_instances = vec![]; + for task_num in 0..params.tasks { + task_instances.push(create_instances_with_groups( + ¶ms, + &opctx, + &db, + &authz_project, + task_num, + ) + .await); + } + + // Each iteration is an "attempt" at the test. + for _ in 0..iterations { + // Within each attempt, we spawn the tasks requested. + let mut set = tokio::task::JoinSet::new(); + + // This barrier exists to lessen the impact of "task spawning" on the benchmark. + // + // We want to have all tasks run as concurrently as possible, since we're trying to + // measure contention explicitly. + let barrier = Arc::new(Barrier::new(params.tasks)); + + for task_num in 0..params.tasks { + set.spawn({ + let opctx = opctx.clone(); + let db = db.clone(); + let barrier = barrier.clone(); + let params = params.clone(); + let instances = task_instances[task_num].clone(); - // The sum of "average time to provision a single vmm" across all tasks. - let all_tasks_duration = set - .join_all() - .await - .into_iter() - .fold(Duration::ZERO, |acc, x| acc + x); - - // The normalized "time to provision a single vmm", across both: - // - The number of vmms reserved by each task, and - // - The number of tasks - // - // As an example, if we provision 10 vmms, and have 5 tasks, and we assume - // that VM provisioning time is exactly one second (contention has no impact, somehow): - // - // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average - // duration of "1 second". - // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds - // (1 second average * 5 tasks). - // - So, we'd increment our "total_duration" by "1 second per vmm", which has been - // normalized cross both the tasks and quantities of vmms. - // - // Why bother doing this? - // - // When we perform this normalization, we can vary the "total vmms provisioned" as well - // as "total tasks" significantly, but easily compare test durations with one another. - // - // For example: if the total number of vmms has no impact on the next provisioning - // request, we should see similar durations for "100 vmms reserved" vs "1 vmm - // reserved". However, if more vmms actually make reservation times slower, we'll see - // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: - total_duration += - average_duration(all_tasks_duration, params.tasks); + async move { + reserve_vmms_and_return_average_duration( + ¶ms, + &opctx, + &db, + &instances, + &barrier, + ) + .await + } + }); } - total_duration - }; - duration + // The sum of "average time to provision a single vmm" across all tasks. + let all_tasks_duration = set + .join_all() + .await + .into_iter() + .fold(Duration::ZERO, |acc, x| acc + x); + + // The normalized "time to provision a single vmm", across both: + // - The number of vmms reserved by each task, and + // - The number of tasks + // + // As an example, if we provision 10 vmms, and have 5 tasks, and we assume + // that VM provisioning time is exactly one second (contention has no impact, somehow): + // + // - Each task would take 10 seconds (10 vmms * 1 second), but would return an average + // duration of "1 second". + // - Across all tasks, we'd see an "all_tasks_duration" of 5 seconds + // (1 second average * 5 tasks). + // - So, we'd increment our "total_duration" by "1 second per vmm", which has been + // normalized cross both the tasks and quantities of vmms. + // + // Why bother doing this? + // + // When we perform this normalization, we can vary the "total vmms provisioned" as well + // as "total tasks" significantly, but easily compare test durations with one another. + // + // For example: if the total number of vmms has no impact on the next provisioning + // request, we should see similar durations for "100 vmms reserved" vs "1 vmm + // reserved". However, if more vmms actually make reservation times slower, we'll see + // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: + total_duration += average_duration(all_tasks_duration, params.tasks); + } + // Destroy all our instance records (and their affinity group memberships) + for instance_ids in task_instances { + destroy_instances_with_groups(&opctx, &db, instance_ids).await; + } + + delete_test_groups(&opctx, &db, ¶ms).await; + total_duration } fn sled_reservation_benchmark(c: &mut Criterion) { @@ -193,43 +410,112 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(harness::setup_db(&logctx.log)); - let mut group = c.benchmark_group("vmm-reservation"); - for tasks in TASK_PARAMS { - for vmms in VMM_PARAMS { - let params = TestParams { vmms, tasks }; - let name = format!("{tasks}-tasks-{vmms}-vmms"); - - // Initialize the harness before calling "bench_function" so - // that the "warm-up" calls to "bench_function" are actually useful - // at warming up the database. - // - // This mitigates any database-caching issues like "loading schema - // on boot", or "connection pooling", as the pool stays the same - // between calls to the benchmark function. - let log = logctx.log.clone(); - let harness = rt.block_on(async move { - const SLED_COUNT: usize = 4; - TestHarness::new(&log, SLED_COUNT).await - }); + let group_patterns = [ + // No Affinity Groups + InstanceGroupPattern { + description: "no-groups", + group_pattern: vec![], + }, + // Alternating "Affinity Group" and "Anti-Affinity Group", both permissive + InstanceGroupPattern { + description: "affinity-groups-permissive", + group_pattern: vec![ + InstanceGroups { + belongs_to: vec![GroupInfo { + name: "affinity-1", + policy: external::AffinityPolicy::Allow, + flavor: GroupType::Affinity, + }], + }, + InstanceGroups { + belongs_to: vec![GroupInfo { + name: "anti-affinity-1", + policy: external::AffinityPolicy::Allow, + flavor: GroupType::AntiAffinity, + }], + }, + ], + }, + // TODO create a test for "policy = Fail" groups? + ]; - // Actually invoke the benchmark. - group.bench_function(&name, |b| { - b.to_async(&rt).iter_custom(|iters| { - let opctx = harness.opctx(); - let db = harness.db(); - async move { bench_reservation(opctx, db, params, iters).await } - }) - }); + for grouping in &group_patterns { + let mut group = c.benchmark_group(format!( + "vmm-reservation-{}", + grouping.description + )); - // Clean-up the harness; we'll use a new database between - // varations in parameters. - rt.block_on(async move { - harness.print_contention().await; - harness.terminate().await; - }); + // Flat sampling is recommended for benchmarks which run "longer than + // milliseconds", which is the case for these benches. + // + // See: https://bheisler.github.io/criterion.rs/book/user_guide/advanced_configuration.html + group.sampling_mode(SamplingMode::Flat); + + for sleds in SLED_PARAMS { + for tasks in TASK_PARAMS { + for vmms in VMM_PARAMS { + let params = TestParams { + vmms, + tasks, + group_pattern: grouping.clone(), + }; + let name = + format!("{sleds}-sleds-{tasks}-tasks-{vmms}-vmms"); + + // TODO: This can also fail depending on the group requirements, + // if we're using policy = fail. Maybe construct "TestParams" + // differently to let each test decide this. + if tasks * vmms > max_resource_request_count(sleds) { + eprintln!( + "{name} would request too many VMMs; skipping..." + ); + continue; + } + + // Initialize the harness before calling "bench_function" so + // that the "warm-up" calls to "bench_function" are actually useful + // at warming up the database. + // + // This mitigates any database-caching issues like "loading schema + // on boot", or "connection pooling", as the pool stays the same + // between calls to the benchmark function. + let log = logctx.log.clone(); + let harness = rt.block_on(async move { + TestHarness::new(&log, sleds).await + }); + + // Actually invoke the benchmark. + group.bench_function(&name, |b| { + b.to_async(&rt).iter_custom(|iters| { + let opctx = harness.opctx(); + let db = harness.db(); + let authz_project = harness.authz_project(); + let params = params.clone(); + async move { + bench_reservation( + opctx, + db, + authz_project, + params, + iters, + ) + .await + } + }) + }); + // Clean-up the harness; we'll use a new database between + // varations in parameters. + rt.block_on(async move { + if std::env::var("SHOW_CONTENTION").is_ok() { + harness.print_contention().await; + } + harness.terminate().await; + }); + } + } } + group.finish(); } - group.finish(); logctx.cleanup_successful(); } @@ -241,6 +527,8 @@ criterion_group!( // - Higher noise threshold, to avoid avoid false positive change detection config = Criterion::default() .sample_size(10) + // Allow for 10% variance in performance without identifying performance + // improvements/regressions .noise_threshold(0.10); targets = sled_reservation_benchmark ); From 9a3ef20fa7e24dee78d9fe8644ef5ca30b4cd5a5 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Feb 2025 12:26:24 -0800 Subject: [PATCH 48/84] fmt --- nexus/db-queries/benches/harness/db_utils.rs | 20 ++-- nexus/db-queries/benches/sled_reservation.rs | 98 ++++++++++---------- 2 files changed, 53 insertions(+), 65 deletions(-) diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index 33fb48c86f7..bd33e31c15c 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -237,14 +237,10 @@ pub async fn delete_affinity_group( .project_name_owned(project.into()) .affinity_group_name_owned(group.into()) .lookup_for(authz::Action::Delete) - .await.unwrap(); + .await + .unwrap(); - db.affinity_group_delete( - &opctx, - &authz_group, - ) - .await - .unwrap(); + db.affinity_group_delete(&opctx, &authz_group).await.unwrap(); } pub async fn create_anti_affinity_group( @@ -284,14 +280,10 @@ pub async fn delete_anti_affinity_group( .project_name_owned(project.into()) .anti_affinity_group_name_owned(group.into()) .lookup_for(authz::Action::Delete) - .await.unwrap(); + .await + .unwrap(); - db.anti_affinity_group_delete( - &opctx, - &authz_group, - ) - .await - .unwrap(); + db.anti_affinity_group_delete(&opctx, &authz_group).await.unwrap(); } pub async fn create_affinity_group_member( diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index f463f92c612..3553557dbb8 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -66,8 +66,14 @@ struct InstanceGroupPattern { description: &'static str, // These "instance group settings" should be applied. // - // NOTE: Currently, we rotate through these groups - group_pattern: Vec, + // This is called "stripe" because we rotate through these groups. + // + // For example, with a stripe.len() == 2... + // - Instance 0 belongs to the groups in stripe[0] + // - Instance 1 belongs to the groups in stripe[1] + // - Instance 2 belongs to the groups in stripe[0] + // - etc + stripe: Vec, } #[derive(Clone)] @@ -110,6 +116,11 @@ fn average_duration(duration: Duration, divisor: usize) -> Duration { ) } +// Test setup: Create all instances and group memberships that will be +// used by a particular task under test. +// +// Precondition: We expect that the test groups (shared by all tasks) +// will already exist. async fn create_instances_with_groups( params: &TestParams, opctx: &OpContext, @@ -129,10 +140,9 @@ async fn create_instances_with_groups( instance_ids.push(instance_id); - let patterns = params.group_pattern.group_pattern.len(); + let patterns = params.group_pattern.stripe.len(); if patterns > 0 { - let groups = - ¶ms.group_pattern.group_pattern[i % patterns].belongs_to; + let groups = ¶ms.group_pattern.stripe[i % patterns].belongs_to; for group in groups { match group.flavor { GroupType::Affinity => { @@ -164,7 +174,10 @@ async fn create_instances_with_groups( instance_ids } -async fn destroy_instances_with_groups( +// Destroys all instances. +// +// This implicitly removes group memberships. +async fn destroy_instances( opctx: &OpContext, db: &DataStore, instances: Vec, @@ -172,8 +185,6 @@ async fn destroy_instances_with_groups( for instance_id in instances { delete_instance_record(opctx, db, instance_id).await; } - - // TODO: Delete groups too } // Reserves "params.vmms" vmms, and later deletes their reservations. @@ -186,7 +197,11 @@ async fn reserve_vmms_and_return_average_duration( instance_ids: &Vec, barrier: &Barrier, ) -> Duration { - assert_eq!(instance_ids.len(), params.vmms, "Not enough instances to provision"); + assert_eq!( + instance_ids.len(), + params.vmms, + "Not enough instances to provision" + ); let mut vmm_ids = Vec::with_capacity(params.vmms); // Wait for all tasks to start at roughly the same time @@ -238,7 +253,7 @@ async fn create_test_groups( ) { let all_groups: HashSet<_> = params .group_pattern - .group_pattern + .stripe .iter() .map(|groups| groups.belongs_to.iter()) .flatten() @@ -276,7 +291,7 @@ async fn delete_test_groups( ) { let all_groups: HashSet<_> = params .group_pattern - .group_pattern + .stripe .iter() .map(|groups| groups.belongs_to.iter()) .flatten() @@ -284,20 +299,10 @@ async fn delete_test_groups( for group in all_groups { match group.flavor { GroupType::Affinity => { - delete_affinity_group( - opctx, - db, - group.name, - ) - .await; + delete_affinity_group(opctx, db, group.name).await; } GroupType::AntiAffinity => { - delete_anti_affinity_group( - opctx, - db, - group.name, - ) - .await; + delete_anti_affinity_group(opctx, db, group.name).await; } } } @@ -312,23 +317,22 @@ async fn bench_reservation( ) -> Duration { let mut total_duration = Duration::ZERO; - // TODO: Can we avoid passing "authz_project" everywhere, and just do - // lookups in db_utils? - - // Create all groups and instances belonging to those groups before - // we actually do any iterations. This is a slight optimization to - // make the benchmarks - focused on VMM reservation time - a little faster. + // Create all groups and instances belonging to those groups before we + // actually do any iterations. This is a slight optimization to make the + // benchmarks - which are focused on VMM reservation time - a little faster. create_test_groups(&opctx, &db, &authz_project, ¶ms).await; let mut task_instances = vec![]; for task_num in 0..params.tasks { - task_instances.push(create_instances_with_groups( - ¶ms, - &opctx, - &db, - &authz_project, - task_num, - ) - .await); + task_instances.push( + create_instances_with_groups( + ¶ms, + &opctx, + &db, + &authz_project, + task_num, + ) + .await, + ); } // Each iteration is an "attempt" at the test. @@ -352,11 +356,7 @@ async fn bench_reservation( async move { reserve_vmms_and_return_average_duration( - ¶ms, - &opctx, - &db, - &instances, - &barrier, + ¶ms, &opctx, &db, &instances, &barrier, ) .await } @@ -397,9 +397,8 @@ async fn bench_reservation( } // Destroy all our instance records (and their affinity group memberships) for instance_ids in task_instances { - destroy_instances_with_groups(&opctx, &db, instance_ids).await; + destroy_instances(&opctx, &db, instance_ids).await; } - delete_test_groups(&opctx, &db, ¶ms).await; total_duration } @@ -412,14 +411,11 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let group_patterns = [ // No Affinity Groups - InstanceGroupPattern { - description: "no-groups", - group_pattern: vec![], - }, + InstanceGroupPattern { description: "no-groups", stripe: vec![] }, // Alternating "Affinity Group" and "Anti-Affinity Group", both permissive InstanceGroupPattern { description: "affinity-groups-permissive", - group_pattern: vec![ + stripe: vec![ InstanceGroups { belongs_to: vec![GroupInfo { name: "affinity-1", @@ -436,7 +432,7 @@ fn sled_reservation_benchmark(c: &mut Criterion) { }, ], }, - // TODO create a test for "policy = Fail" groups? + // TODO create a test for "policy = Fail" groups. ]; for grouping in &group_patterns { @@ -462,7 +458,7 @@ fn sled_reservation_benchmark(c: &mut Criterion) { let name = format!("{sleds}-sleds-{tasks}-tasks-{vmms}-vmms"); - // TODO: This can also fail depending on the group requirements, + // NOTE: This can also fail depending on the group requirements, // if we're using policy = fail. Maybe construct "TestParams" // differently to let each test decide this. if tasks * vmms > max_resource_request_count(sleds) { From f02835183087122c7cd8abd9e78f4a66a582f6ec Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Feb 2025 12:30:59 -0800 Subject: [PATCH 49/84] clippy --- nexus/db-queries/benches/harness/db_utils.rs | 2 +- nexus/db-queries/benches/sled_reservation.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index bd33e31c15c..dad09beeaa0 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -149,7 +149,7 @@ pub async fn create_instance_record( description: "".to_string(), }, ncpus: 2i64.try_into().unwrap(), - memory: external::ByteCount::from_gibibytes_u32(16).into(), + memory: external::ByteCount::from_gibibytes_u32(16), hostname: "myhostname".try_into().unwrap(), user_data: Vec::new(), network_interfaces: diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 3553557dbb8..f4ef4741029 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -255,8 +255,7 @@ async fn create_test_groups( .group_pattern .stripe .iter() - .map(|groups| groups.belongs_to.iter()) - .flatten() + .flat_map(|groups| groups.belongs_to.iter()) .collect(); for group in all_groups { match group.flavor { @@ -293,8 +292,7 @@ async fn delete_test_groups( .group_pattern .stripe .iter() - .map(|groups| groups.belongs_to.iter()) - .flatten() + .flat_map(|groups| groups.belongs_to.iter()) .collect(); for group in all_groups { match group.flavor { From 2ea4dea75e6eb8b9132a08a7aba66c2f10b0c13f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Feb 2025 14:49:48 -0800 Subject: [PATCH 50/84] Remove unused test output --- .../output/lookup_affinity_sleds_query.sql | 28 ----------------- .../lookup_anti_affinity_sleds_query.sql | 30 ------------------- 2 files changed, 58 deletions(-) delete mode 100644 nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql delete mode 100644 nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql diff --git a/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql deleted file mode 100644 index b03bbc45fcf..00000000000 --- a/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql +++ /dev/null @@ -1,28 +0,0 @@ -WITH - our_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $1), - other_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_groups ON affinity_group_instance_membership.group_id = our_groups.group_id - WHERE - instance_id != $2 - ), - other_instances_by_policy - AS ( - SELECT - policy, instance_id - FROM - other_instances - JOIN affinity_group ON - affinity_group.id = other_instances.group_id AND affinity_group.failure_domain = 'sled' - WHERE - affinity_group.time_deleted IS NULL - ) -SELECT - DISTINCT policy, sled_id -FROM - other_instances_by_policy - JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_instances_by_policy.instance_id diff --git a/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql deleted file mode 100644 index ec81dfbcf14..00000000000 --- a/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql +++ /dev/null @@ -1,30 +0,0 @@ -WITH - our_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $1), - other_instances - AS ( - SELECT - anti_affinity_group_instance_membership.group_id, instance_id - FROM - anti_affinity_group_instance_membership - JOIN our_groups ON anti_affinity_group_instance_membership.group_id = our_groups.group_id - WHERE - instance_id != $2 - ), - other_instances_by_policy - AS ( - SELECT - policy, instance_id - FROM - other_instances - JOIN anti_affinity_group ON - anti_affinity_group.id = other_instances.group_id - AND anti_affinity_group.failure_domain = 'sled' - WHERE - anti_affinity_group.time_deleted IS NULL - ) -SELECT - DISTINCT policy, sled_id -FROM - other_instances_by_policy - JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_instances_by_policy.instance_id From 3edc5ed0647c47ee5547abe5ba9f3053b8728d51 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 18 Feb 2025 13:48:45 -0800 Subject: [PATCH 51/84] Fix mismerge --- nexus/tests/integration_tests/schema.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index dac092f35fc..8be321f008e 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -1685,7 +1685,7 @@ fn after_107_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { }) } -fn before_124_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn before_124_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { // Insert a region snapshot replacement record let request_id: Uuid = @@ -1697,7 +1697,7 @@ fn before_124_0_0(client: &Client) -> BoxFuture<'_, ()> { let snapshot_id: Uuid = "0b8382de-d787-450a-8516-235f33eb0946".parse().unwrap(); - client + ctx.client .batch_execute(&format!( " INSERT INTO region_snapshot_replacement ( @@ -1729,9 +1729,10 @@ fn before_124_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } -fn after_124_0_0(client: &Client) -> BoxFuture<'_, ()> { +fn after_124_0_0<'a>(ctx: &'a MigrationContext<'a>) -> BoxFuture<'a, ()> { Box::pin(async { - let rows = client + let rows = ctx + .client .query( "SELECT replacement_type FROM region_snapshot_replacement;", &[], From 3bf35a2286ef24a3ee2ae0c1049f5116ac8531e0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 20 Feb 2025 16:44:30 -0800 Subject: [PATCH 52/84] Basic implementation of affinity group member within anti-affinity group --- common/src/api/external/mod.rs | 5 + nexus/db-model/src/affinity.rs | 28 ++ nexus/db-model/src/schema.rs | 7 + nexus/db-queries/src/db/datastore/affinity.rs | 292 +++++++++++-- nexus/db-queries/src/db/datastore/sled.rs | 386 ++++++++++++++++++ .../src/db/queries/sled_reservation.rs | 144 +++++-- .../tests/output/sled_find_targets_query.sql | 80 +++- .../output/sled_insert_resource_query.sql | 80 +++- schema/crdb/dbinit.sql | 19 + 9 files changed, 946 insertions(+), 95 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index b0ef9d23884..8b70a4228a5 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -22,6 +22,7 @@ use dropshot::HttpError; pub use dropshot::PaginationOrder; pub use error::*; use futures::stream::BoxStream; +use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use oxnet::IpNet; @@ -1391,12 +1392,16 @@ impl SimpleIdentity for AffinityGroupMember { pub enum AntiAffinityGroupMember { /// An instance belonging to this group, identified by UUID. Instance(InstanceUuid), + + /// An affinity group belonging to this group, identified by UUID. + AffinityGroup(AffinityGroupUuid), } impl SimpleIdentity for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), + AntiAffinityGroupMember::AffinityGroup(id) => *id.as_untyped_uuid(), } } } diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index 5309ac95275..89214f85778 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -11,6 +11,7 @@ use super::Name; use crate::schema::affinity_group; use crate::schema::affinity_group_instance_membership; use crate::schema::anti_affinity_group; +use crate::schema::anti_affinity_group_affinity_membership; use crate::schema::anti_affinity_group_instance_membership; use crate::typed_uuid::DbTypedUuid; use chrono::{DateTime, Utc}; @@ -257,3 +258,30 @@ impl From Self::Instance(member.instance_id.into()) } } + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = anti_affinity_group_affinity_membership)] +pub struct AntiAffinityGroupAffinityMembership { + pub anti_affinity_group_id: DbTypedUuid, + pub affinity_group_id: DbTypedUuid, +} + +impl AntiAffinityGroupAffinityMembership { + pub fn new( + anti_affinity_group_id: AntiAffinityGroupUuid, + affinity_group_id: AffinityGroupUuid, + ) -> Self { + Self { + anti_affinity_group_id: anti_affinity_group_id.into(), + affinity_group_id: affinity_group_id.into(), + } + } +} + +impl From + for external::AntiAffinityGroupMember +{ + fn from(member: AntiAffinityGroupAffinityMembership) -> Self { + Self::AffinityGroup(member.affinity_group_id.into()) + } +} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 433a8979f1d..139374eb16f 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -510,6 +510,13 @@ table! { } } +table! { + anti_affinity_group_affinity_membership (anti_affinity_group_id, affinity_group_id) { + anti_affinity_group_id -> Uuid, + affinity_group_id -> Uuid, + } +} + table! { metric_producer (id) { id -> Uuid, diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 3fe3660e7dd..56caf545e5e 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -18,6 +18,7 @@ use crate::db::model::AffinityGroup; use crate::db::model::AffinityGroupInstanceMembership; use crate::db::model::AffinityGroupUpdate; use crate::db::model::AntiAffinityGroup; +use crate::db::model::AntiAffinityGroupAffinityMembership; use crate::db::model::AntiAffinityGroupInstanceMembership; use crate::db::model::AntiAffinityGroupUpdate; use crate::db::model::Name; @@ -433,27 +434,60 @@ impl DataStore { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; let conn = self.pool_connection_authorized(opctx).await?; - let instance_id = match member { - external::AntiAffinityGroupMember::Instance(id) => id, - }; - - use db::schema::anti_affinity_group_instance_membership::dsl; - dsl::anti_affinity_group_instance_membership - .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) - .select(AntiAffinityGroupInstanceMembership::as_select()) - .get_result_async(&*conn) - .await - .map(|m| m.into()) - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::AntiAffinityGroupMember, - LookupType::by_id(instance_id.into_untyped_uuid()), - ), - ) - }) + match member { + external::AntiAffinityGroupMember::Instance(instance_id) => { + use db::schema::anti_affinity_group_instance_membership::dsl; + dsl::anti_affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter( + dsl::instance_id.eq(instance_id.into_untyped_uuid()), + ) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id( + instance_id.into_untyped_uuid(), + ), + ), + ) + }) + } + external::AntiAffinityGroupMember::AffinityGroup( + affinity_group_id, + ) => { + use db::schema::anti_affinity_group_affinity_membership::dsl; + dsl::anti_affinity_group_affinity_membership + .filter( + dsl::anti_affinity_group_id + .eq(authz_anti_affinity_group.id()), + ) + .filter( + dsl::affinity_group_id + .eq(affinity_group_id.into_untyped_uuid()), + ) + .select(AntiAffinityGroupAffinityMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id( + affinity_group_id.into_untyped_uuid(), + ), + ), + ) + }) + } + } } pub async fn affinity_group_member_add( @@ -593,13 +627,35 @@ impl DataStore { .authorize(authz::Action::Modify, authz_anti_affinity_group) .await?; - let instance_id = match member { - external::AntiAffinityGroupMember::Instance(id) => id, - }; + match member { + external::AntiAffinityGroupMember::Instance(id) => { + self.anti_affinity_group_member_add_instance( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + external::AntiAffinityGroupMember::AffinityGroup(id) => { + self.anti_affinity_group_member_add_group( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + } + } + async fn anti_affinity_group_member_add_instance( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + instance_id: InstanceUuid, + ) -> Result<(), Error> { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_add") + self.transaction_retry_wrapper("anti_affinity_group_member_add_instance") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -703,6 +759,95 @@ impl DataStore { Ok(()) } + async fn anti_affinity_group_member_add_group( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + affinity_group_id: AffinityGroupUuid, + ) -> Result<(), Error> { + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_add_group") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as anti_affinity_group_dsl; + use db::schema::affinity_group::dsl as affinity_group_dsl; + use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + + async move { + // Check that the anti-affinity group exists + anti_affinity_group_dsl::anti_affinity_group + .filter(anti_affinity_group_dsl::time_deleted.is_null()) + .filter(anti_affinity_group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(anti_affinity_group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the affinity group exists + affinity_group_dsl::affinity_group + .filter(affinity_group_dsl::time_deleted.is_null()) + .filter(affinity_group_dsl::id.eq(affinity_group_id.into_untyped_uuid())) + .select(affinity_group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AffinityGroup, + LookupType::ById(affinity_group_id.into_untyped_uuid()) + ), + ) + }) + })?; + + // TODO: It's possible that the affinity group has members + // which are already running. We should probably check this, + // and prevent it, otherwise we could circumvent "policy = + // fail" stances. + + diesel::insert_into(membership_dsl::anti_affinity_group_affinity_membership) + .values(AntiAffinityGroupAffinityMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), + affinity_group_id, + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroupMember, + &affinity_group_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + pub async fn instance_affinity_group_memberships_delete( &self, opctx: &OpContext, @@ -810,13 +955,38 @@ impl DataStore { .authorize(authz::Action::Modify, authz_anti_affinity_group) .await?; - let instance_id = match member { - external::AntiAffinityGroupMember::Instance(id) => id, - }; + match member { + external::AntiAffinityGroupMember::Instance(id) => { + self.anti_affinity_group_instance_member_delete( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + external::AntiAffinityGroupMember::AffinityGroup(id) => { + self.anti_affinity_group_affinity_member_delete( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + } + } + // Deletes an anti-affinity member, when that member is an instance + // + // See: [`Self::anti_affinity_group_member_delete`] + async fn anti_affinity_group_instance_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + instance_id: InstanceUuid, + ) -> Result<(), Error> { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_delete") + self.transaction_retry_wrapper("anti_affinity_group_instance_member_delete") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -868,6 +1038,70 @@ impl DataStore { })?; Ok(()) } + + // Deletes an anti-affinity member, when that member is an affinity group + // + // See: [`Self::anti_affinity_group_member_delete`] + async fn anti_affinity_group_affinity_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + affinity_group_id: AffinityGroupUuid, + ) -> Result<(), Error> { + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_affinity_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + + async move { + // Check that the anti-affinity group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::anti_affinity_group_affinity_membership) + .filter(membership_dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id())) + .filter(membership_dsl::affinity_group_id.eq(affinity_group_id.into_untyped_uuid())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(affinity_group_id.into_untyped_uuid()).into_not_found( + ResourceType::AntiAffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } } #[cfg(test)] diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 227f258c174..815d88e2490 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -1479,6 +1479,31 @@ pub(in crate::db::datastore) mod test { .unwrap(); } + async fn add_affinity_group_to_anti_affinity_group( + datastore: &DataStore, + aa_group_id: AntiAffinityGroupUuid, + a_group_id: AffinityGroupUuid, + ) { + use db::model::AntiAffinityGroupAffinityMembership; + use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + + diesel::insert_into( + membership_dsl::anti_affinity_group_affinity_membership, + ) + .values(AntiAffinityGroupAffinityMembership::new( + aa_group_id, + a_group_id, + )) + .on_conflict(( + membership_dsl::anti_affinity_group_id, + membership_dsl::affinity_group_id, + )) + .do_nothing() + .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap(); + } + async fn create_affinity_group( opctx: &OpContext, datastore: &DataStore, @@ -1609,6 +1634,42 @@ pub(in crate::db::datastore) mod test { Self { id_by_name } } + + async fn add_affinity_groups_to_anti_affinity_group( + &self, + datastore: &DataStore, + aa_group: &'static str, + a_groups: &[&'static str], + ) { + let (affinity, aa_group_id) = + self.id_by_name.get(aa_group).unwrap_or_else(|| { + panic!( + "Anti-affinity group {aa_group} not part of AllGroups" + ) + }); + assert!( + matches!(affinity, Affinity::Negative), + "{aa_group} should be an anti-affinity group, but is not" + ); + + for a_group in a_groups { + let (affinity, a_group_id) = + self.id_by_name.get(a_group).unwrap_or_else(|| { + panic!("Affinity group {a_group} not part of AllGroups") + }); + assert!( + matches!(affinity, Affinity::Positive), + "{a_group} should be an affinity group, but is not" + ); + + add_affinity_group_to_anti_affinity_group( + datastore, + AntiAffinityGroupUuid::from_untyped_uuid(*aa_group_id), + AffinityGroupUuid::from_untyped_uuid(*a_group_id), + ) + .await; + } + } } struct Instance { @@ -2323,6 +2384,331 @@ pub(in crate::db::datastore) mod test { logctx.cleanup_successful(); } + #[tokio::test] + async fn anti_affinity_group_containing_affinity_groups() { + let logctx = dev::test_setup_log( + "anti_affinity_group_containing_affinity_groups", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "anti-affinity", + &["affinity1", "affinity2"], + ) + .await; + + // The instances on sled 0 and 1 belong to "affinity1" + // The instances on sled 2 and 3 belong to "affinity2" + // + // A new instance in "affinity2" can only be placed on sled 2. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity1").sled(sleds[1].id()), + Instance::new().group("affinity2").sled(sleds[2].id()), + Instance::new() + .group("affinity2") + .sled(sleds[3].id()) + .use_many_resources(), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("affinity2"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!( + resource.sled_id.into_untyped_uuid(), + sleds[2].id(), + "All sleds: {sled_ids:#?}", + sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_containing_affinity_groups_force_away_from_affine( + ) { + let logctx = dev::test_setup_log("anti_affinity_group_containing_affinity_groups_force_away_from_affine"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 5; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "anti-affinity", + &["affinity1", "affinity2"], + ) + .await; + + // The instances on sled 0 and 1 belong to "affinity1" + // The instance on sled 2 belongs to "affinity2" + // The instance on sled 3 directly belongs to "anti-affinity" + // + // Even though the instance belongs to the affinity groups - and + // would "prefer" to be co-located with them - it's forced to be placed + // on sleds[4], since that's the only sled which doesn't violate + // the hard "anti-affinity" requirement. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity1").sled(sleds[1].id()), + Instance::new().group("affinity2").sled(sleds[2].id()), + Instance::new().group("anti-affinity").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + // This instance is not part of "anti-affinity" directly. However, it + // is indirectly part of the anti-affinity group, because of its + // affinity group memberships. + let test_instance = + Instance::new().group("affinity1").group("affinity2"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!( + resource.sled_id.into_untyped_uuid(), + sleds[4].id(), + "All sleds: {sled_ids:#?}", + sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_containing_affinity_groups_multigroup() { + let logctx = dev::test_setup_log( + "anti_affinity_group_containing_affinity_groups_multigroup", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "wolf-eat-goat", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Negative, + name: "goat-eat-cabbage", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "wolf", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "goat", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "cabbage", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "wolf-eat-goat", + &["wolf", "goat"], + ) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "goat-eat-cabbage", + &["goat", "cabbage"], + ) + .await; + + let instances = [ + Instance::new().group("wolf").sled(sleds[0].id()), + Instance::new().group("cabbage").sled(sleds[1].id()), + Instance::new().group("goat").sled(sleds[2].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("wolf").group("cabbage"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + // This instance can go on either sled[0] or sled[1], but not sled[2] + assert_ne!( + resource.sled_id.into_untyped_uuid(), + sleds[2].id(), + "All sleds: {sled_ids:#?}", + sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_containing_overlapping_affinity_groups() { + let logctx = dev::test_setup_log( + "anti_affinity_group_containing_overlapping_affinity_groups", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "anti-affinity", + &["affinity1", "affinity2"], + ) + .await; + + // The instances on sled 0 and 1 belong to "affinity1" + // The instances on sled 2 and 3 belong to "affinity2" + // + // If a new instance belongs to both affinity groups, it cannot satisfy + // the anti-affinity requirements. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity1").sled(sleds[1].id()), + Instance::new().group("affinity2").sled(sleds[2].id()), + Instance::new().group("affinity2").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("affinity1").group("affinity2"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + + let SledReservationTransactionError::Reservation( + SledReservationError::NotFound, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn affinity_multi_group() { let logctx = dev::test_setup_log("affinity_multi_group"); diff --git a/nexus/db-queries/src/db/queries/sled_reservation.rs b/nexus/db-queries/src/db/queries/sled_reservation.rs index 912fd644da4..7fe28101a79 100644 --- a/nexus/db-queries/src/db/queries/sled_reservation.rs +++ b/nexus/db-queries/src/db/queries/sled_reservation.rs @@ -57,18 +57,70 @@ pub fn sled_find_targets_query( COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " ).param().sql(" <= sled.reservoir_size ), - our_aa_groups AS ( + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), + our_direct_aa_groups AS ( + -- Anti-affinity groups to which our instance belongs SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = ").param().sql(" ), - other_aa_instances AS ( + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id,instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE instance_id != ").param().sql(" ), + our_indirect_aa_groups AS ( + -- Anti-affinity groups to which our instance's affinity groups belong + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN COUNT(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN TRUE + ELSE FALSE + END AS exactly_one_affinity_group + FROM anti_affinity_group_affinity_membership + WHERE affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + -- If our instance belongs to exactly one of these groups... + CASE WHEN our_indirect_aa_groups.exactly_one_affinity_group + -- ... exclude that group from our anti-affinity + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + -- ... otherwise, consider all groups anti-affine + ELSE TRUE + END + ), + other_aa_instances AS ( + SELECT * FROM other_direct_aa_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_groups + ), other_aa_instances_by_policy AS ( SELECT policy,instance_id FROM other_aa_instances @@ -85,18 +137,6 @@ pub fn sled_find_targets_query( ON sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id ), - our_a_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_a_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_a_groups - ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE instance_id != ").param().sql(" - ), other_a_instances_by_policy AS ( SELECT policy,instance_id FROM other_a_instances @@ -172,18 +212,70 @@ pub fn sled_insert_resource_query( COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " ).param().sql(" <= sled.reservoir_size ), - our_aa_groups AS ( + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), + our_direct_aa_groups AS ( + -- Anti-affinity groups to which our instance belongs SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = ").param().sql(" ), - other_aa_instances AS ( + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id,instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE instance_id != ").param().sql(" ), + our_indirect_aa_groups AS ( + -- Anti-affinity groups to which our instance's affinity groups belong + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN COUNT(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN TRUE + ELSE FALSE + END AS only_one_affinity_group + FROM anti_affinity_group_affinity_membership + WHERE affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + -- If our instance belongs to exactly one of these groups... + CASE WHEN our_indirect_aa_groups.only_one_affinity_group + -- ... exclude that group from our anti-affinity + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + -- ... otherwise, consider all groups anti-affine + ELSE TRUE + END + ), + other_aa_instances AS ( + SELECT * FROM other_direct_aa_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_groups + ), banned_instances AS ( SELECT instance_id FROM other_aa_instances @@ -201,18 +293,6 @@ pub fn sled_insert_resource_query( ON sled_resource_vmm.instance_id = banned_instances.instance_id ), - our_a_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_a_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_a_groups - ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE instance_id != ").param().sql(" - ), required_instances AS ( SELECT policy,instance_id FROM other_a_instances diff --git a/nexus/db-queries/tests/output/sled_find_targets_query.sql b/nexus/db-queries/tests/output/sled_find_targets_query.sql index 9de50f47f7f..f230a4b6dcb 100644 --- a/nexus/db-queries/tests/output/sled_find_targets_query.sql +++ b/nexus/db-queries/tests/output/sled_find_targets_query.sql @@ -17,18 +17,75 @@ WITH AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $3 <= sled.reservoir_size ), - our_aa_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $4), - other_aa_instances + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $4), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $5 + ), + our_direct_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $6), + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id, instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups ON - anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE - instance_id != $5 + instance_id != $7 + ), + our_indirect_aa_groups + AS ( + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN count(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN true + ELSE false + END + AS exactly_one_affinity_group + FROM + anti_affinity_group_affinity_membership + WHERE + affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + anti_affinity_group_instance_membership.group_id + = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + CASE + WHEN our_indirect_aa_groups.exactly_one_affinity_group + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + ELSE true + END + ), + other_aa_instances + AS ( + SELECT * FROM other_direct_aa_instances + UNION SELECT * FROM other_indirect_aa_instances_via_instances + UNION SELECT * FROM other_indirect_aa_instances_via_groups ), other_aa_instances_by_policy AS ( @@ -51,17 +108,6 @@ WITH JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id ), - our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $6), - other_a_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE - instance_id != $7 - ), other_a_instances_by_policy AS ( SELECT diff --git a/nexus/db-queries/tests/output/sled_insert_resource_query.sql b/nexus/db-queries/tests/output/sled_insert_resource_query.sql index 9cfb68e3008..a0ae17555dd 100644 --- a/nexus/db-queries/tests/output/sled_insert_resource_query.sql +++ b/nexus/db-queries/tests/output/sled_insert_resource_query.sql @@ -20,18 +20,75 @@ WITH AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $4 <= sled.reservoir_size ), - our_aa_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $5), - other_aa_instances + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $5), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $6 + ), + our_direct_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $7), + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id, instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups ON - anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE - instance_id != $6 + instance_id != $8 + ), + our_indirect_aa_groups + AS ( + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN count(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN true + ELSE false + END + AS only_one_affinity_group + FROM + anti_affinity_group_affinity_membership + WHERE + affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + anti_affinity_group_instance_membership.group_id + = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + CASE + WHEN our_indirect_aa_groups.only_one_affinity_group + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + ELSE true + END + ), + other_aa_instances + AS ( + SELECT * FROM other_direct_aa_instances + UNION SELECT * FROM other_indirect_aa_instances_via_instances + UNION SELECT * FROM other_indirect_aa_instances_via_groups ), banned_instances AS ( @@ -54,17 +111,6 @@ WITH banned_instances JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = banned_instances.instance_id ), - our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $7), - other_a_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE - instance_id != $8 - ), required_instances AS ( SELECT diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index ce6e4ccb2ec..f788b1f610e 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4214,6 +4214,25 @@ CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_ins instance_id ); +-- Describes an affinity group's membership within an anti-affinity group +-- +-- Since the naming here is a little confusing: +-- This allows an anti-affinity group to contain affinity groups as members. +-- This is useful for saying "I want these groups of VMMs to be anti-affine from +-- one another", rather than "I want individual VMMs to be anti-affine from each other". +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_affinity_membership ( + anti_affinity_group_id UUID NOT NULL, + affinity_group_id UUID NOT NULL, + + PRIMARY KEY (anti_affinity_group_id, affinity_group_id) +); + +-- We need to look up all memberships of an affinity group so we can revoke these +-- memberships efficiently when affinity groups are deleted +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_affinity_membership_by_affinity_group ON omicron.public.anti_affinity_group_affinity_membership ( + affinity_group_id +); + -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, From daeb2e1a08b5c6cfe1e3fc81d8a848717d3c5b1b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Feb 2025 11:42:19 -0800 Subject: [PATCH 53/84] Update API --- openapi/nexus.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/openapi/nexus.json b/openapi/nexus.json index 17088ed5c9c..ec00fdab967 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12504,6 +12504,25 @@ "type", "value" ] + }, + { + "description": "An affinity group belonging to this group, identified by UUID.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "affinity_group" + ] + }, + "value": { + "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" + } + }, + "required": [ + "type", + "value" + ] } ] }, @@ -23053,6 +23072,10 @@ } } }, + "TypedUuidForAffinityGroupKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForInstanceKind": { "type": "string", "format": "uuid" From 2d6504804b8f8bbb2d53512fafa0f74c037d21fb Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Feb 2025 13:00:58 -0800 Subject: [PATCH 54/84] Schema migration for db updates --- nexus/db-model/src/schema_versions.rs | 3 ++- schema/crdb/anti-affinity-group-affinity-member/up01.sql | 6 ++++++ schema/crdb/anti-affinity-group-affinity-member/up02.sql | 4 ++++ schema/crdb/dbinit.sql | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 schema/crdb/anti-affinity-group-affinity-member/up01.sql create mode 100644 schema/crdb/anti-affinity-group-affinity-member/up02.sql diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 6ba30a87623..c59975f9ee0 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(127, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(128, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(128, "anti-affinity-group-affinity-member"), KnownVersion::new(127, "sled-resource-for-vmm"), KnownVersion::new(126, "affinity"), KnownVersion::new(125, "blueprint-disposition-expunged-cleanup"), diff --git a/schema/crdb/anti-affinity-group-affinity-member/up01.sql b/schema/crdb/anti-affinity-group-affinity-member/up01.sql new file mode 100644 index 00000000000..4e16d622466 --- /dev/null +++ b/schema/crdb/anti-affinity-group-affinity-member/up01.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_affinity_membership ( + anti_affinity_group_id UUID NOT NULL, + affinity_group_id UUID NOT NULL, + + PRIMARY KEY (anti_affinity_group_id, affinity_group_id) +); diff --git a/schema/crdb/anti-affinity-group-affinity-member/up02.sql b/schema/crdb/anti-affinity-group-affinity-member/up02.sql new file mode 100644 index 00000000000..459c36df1bb --- /dev/null +++ b/schema/crdb/anti-affinity-group-affinity-member/up02.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_affinity_membership_by_affinity_group ON omicron.public.anti_affinity_group_affinity_membership ( + affinity_group_id +); + diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e5291893e7a..402ab650efb 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4977,7 +4977,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '127.0.0', NULL) + (TRUE, NOW(), NOW(), '128.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; From 4e6362de35618a8636ef9bdc16e3ad8de98ac446 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 24 Feb 2025 08:07:43 -0800 Subject: [PATCH 55/84] boilerplate, gotta fix pagination on list next --- nexus/db-queries/src/db/datastore/affinity.rs | 81 ++++++++-- nexus/external-api/src/lib.rs | 42 +++++- nexus/src/app/affinity.rs | 81 +++++++++- nexus/src/external_api/http_entrypoints.rs | 142 +++++++++++++++++- nexus/types/src/external_api/params.rs | 6 + 5 files changed, 329 insertions(+), 23 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 56caf545e5e..e07239fdffa 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -365,7 +365,7 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - pub async fn anti_affinity_group_member_list( + pub async fn anti_affinity_group_member_instance_list( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, @@ -373,6 +373,13 @@ impl DataStore { ) -> ListResultVec { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + // TODO: Need to also look up "group_membership" here + // TODO: definitely test listing both? + // TODO: atlernately - make this API "instance member" specific, make a + // different one for "affinity group members", and paginate over both. + // + // That might be preferable. + use db::schema::anti_affinity_group_instance_membership::dsl; match pagparams { PaginatedBy::Id(pagparams) => paginated( @@ -1804,7 +1811,11 @@ mod tests { }; let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -1832,7 +1843,11 @@ mod tests { // We should now be able to list the new member let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert_eq!(members.len(), 1); @@ -1855,7 +1870,11 @@ mod tests { .await .unwrap(); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2036,7 +2055,11 @@ mod tests { }; let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2100,7 +2123,11 @@ mod tests { // We should now be able to list the new member let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert_eq!(members.len(), 1); @@ -2124,7 +2151,11 @@ mod tests { .await .unwrap(); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2242,7 +2273,11 @@ mod tests { }; let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2274,7 +2309,11 @@ mod tests { // Confirm that no instance members exist let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2401,7 +2440,11 @@ mod tests { }; let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2438,7 +2481,11 @@ mod tests { // Confirm that no instance members exist let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2974,7 +3021,11 @@ mod tests { }; let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert_eq!(members.len(), 1); @@ -3012,7 +3063,11 @@ mod tests { ); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_instance_list( + &opctx, + &authz_group, + &pagbyid, + ) .await .unwrap(); assert!(members.is_empty()); diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index c843090eddc..0f6b23286dc 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -1409,7 +1409,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result>, HttpError>; - /// Fetch an anti-affinity group member + /// Fetch an anti-affinity group member (where that member is an instance) #[endpoint { method = GET, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1421,7 +1421,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Add a member to an anti-affinity group + /// Add a member to an anti-affinity group (where that member is an instance) #[endpoint { method = POST, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1433,7 +1433,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Remove a member from an anti-affinity group + /// Remove a member from an anti-affinity group (where that member is an instance) #[endpoint { method = DELETE, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1445,6 +1445,42 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result; + /// Fetch an anti-affinity group member (where that member is an affinity group) + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an anti-affinity group (where that member is an affinity group) + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_affinity_group_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an anti-affinity group (where that member is an affinity group) + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + /// Create an anti-affinity group #[endpoint { method = POST, diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index ca8d0c708cd..11c2fb10cea 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -21,6 +21,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; @@ -260,7 +261,7 @@ impl super::Nexus { .await?; Ok(self .db_datastore - .anti_affinity_group_member_list( + .anti_affinity_group_member_instance_list( opctx, &authz_anti_affinity_group, pagparams, @@ -290,7 +291,7 @@ impl super::Nexus { .await } - pub(crate) async fn anti_affinity_group_member_view( + pub(crate) async fn anti_affinity_group_member_instance_view( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, @@ -313,6 +314,29 @@ impl super::Nexus { .await } + pub(crate) async fn anti_affinity_group_member_affinity_group_view( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = + anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + ); + + self.db_datastore + .anti_affinity_group_member_view( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } + pub(crate) async fn affinity_group_member_add( &self, opctx: &OpContext, @@ -337,7 +361,7 @@ impl super::Nexus { Ok(member) } - pub(crate) async fn anti_affinity_group_member_add( + pub(crate) async fn anti_affinity_group_member_instance_add( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, @@ -362,6 +386,31 @@ impl super::Nexus { Ok(member) } + pub(crate) async fn anti_affinity_group_member_affinity_group_add( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + ); + + self.db_datastore + .anti_affinity_group_member_add( + opctx, + &authz_anti_affinity_group, + member.clone(), + ) + .await?; + Ok(member) + } + pub(crate) async fn affinity_group_member_delete( &self, opctx: &OpContext, @@ -381,7 +430,7 @@ impl super::Nexus { .await } - pub(crate) async fn anti_affinity_group_member_delete( + pub(crate) async fn anti_affinity_group_member_instance_delete( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, @@ -404,4 +453,28 @@ impl super::Nexus { ) .await } + + pub(crate) async fn anti_affinity_group_member_affinity_group_delete( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + ) -> Result<(), Error> { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + ); + + self.db_datastore + .anti_affinity_group_member_delete( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index b1cf5b59e5a..d9298f1af36 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -2983,7 +2983,7 @@ impl NexusExternalApi for NexusExternalApiImpl { nexus.instance_lookup(&opctx, instance_selector)?; let group = nexus - .anti_affinity_group_member_view( + .anti_affinity_group_member_instance_view( &opctx, &group_lookup, &instance_lookup, @@ -3029,7 +3029,7 @@ impl NexusExternalApi for NexusExternalApiImpl { nexus.instance_lookup(&opctx, instance_selector)?; let member = nexus - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &group_lookup, &instance_lookup, @@ -3074,7 +3074,7 @@ impl NexusExternalApi for NexusExternalApiImpl { nexus.instance_lookup(&opctx, instance_selector)?; nexus - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( &opctx, &group_lookup, &instance_lookup, @@ -3089,6 +3089,142 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + async fn anti_affinity_group_member_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + // Select anti-affinity group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select affinity group + let affinity_group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let affinity_group_lookup = + nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; + + let group = nexus + .anti_affinity_group_member_affinity_group_view( + &opctx, + &group_lookup, + &affinity_group_lookup, + ) + .await?; + + Ok(HttpResponseOk(group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_affinity_group_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select anti-affinity group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select affinity group + let affinity_group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let affinity_group_lookup = + nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; + + let member = nexus + .anti_affinity_group_member_affinity_group_add( + &opctx, + &group_lookup, + &affinity_group_lookup, + ) + .await?; + Ok(HttpResponseCreated(member)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select anti-affinity group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select affinity group + let affinity_group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let affinity_group_lookup = + nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; + + nexus + .anti_affinity_group_member_affinity_group_delete( + &opctx, + &group_lookup, + &affinity_group_lookup, + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn anti_affinity_group_create( rqctx: RequestContext, query_params: Query, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index e87acd4e590..5614f25c3dc 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -839,6 +839,12 @@ pub struct AntiAffinityInstanceGroupMemberPath { pub instance: NameOrId, } +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AntiAffinityAffinityGroupMemberPath { + pub anti_affinity_group: NameOrId, + pub affinity_group: NameOrId, +} + /// Create-time parameters for an `AntiAffinityGroup` #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct AntiAffinityGroupCreate { From f50749daa9d945be7a64ca0dadc18d13681a329c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 24 Feb 2025 11:37:34 -0800 Subject: [PATCH 56/84] Manual pagination of both tables at once --- common/src/api/external/mod.rs | 8 +- nexus/db-model/src/schema.rs | 1 + nexus/db-queries/src/db/datastore/affinity.rs | 181 ++++++++---------- nexus/src/app/affinity.rs | 13 +- nexus/src/external_api/http_entrypoints.rs | 3 +- 5 files changed, 95 insertions(+), 111 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 8b70a4228a5..b422b6d85ff 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1390,18 +1390,18 @@ impl SimpleIdentity for AffinityGroupMember { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { - /// An instance belonging to this group, identified by UUID. - Instance(InstanceUuid), - /// An affinity group belonging to this group, identified by UUID. AffinityGroup(AffinityGroupUuid), + + /// An instance belonging to this group, identified by UUID. + Instance(InstanceUuid), } impl SimpleIdentity for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { - AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), AntiAffinityGroupMember::AffinityGroup(id) => *id.as_untyped_uuid(), + AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), } } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 954f1d0d0b7..8a7c43f5f82 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -2079,6 +2079,7 @@ allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( anti_affinity_group, + anti_affinity_group_affinity_membership, anti_affinity_group_instance_membership, affinity_group, affinity_group_instance_membership, diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index e07239fdffa..d5efeb56925 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -24,6 +24,7 @@ use crate::db::model::AntiAffinityGroupUpdate; use crate::db::model::Name; use crate::db::model::Project; use crate::db::pagination::paginated; +use crate::db::raw_query_builder::QueryBuilder; use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; @@ -31,6 +32,7 @@ use diesel::prelude::*; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; @@ -42,6 +44,7 @@ use omicron_uuid_kinds::AntiAffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use ref_cast::RefCast; +use uuid::Uuid; impl DataStore { pub async fn affinity_group_list( @@ -365,39 +368,76 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - pub async fn anti_affinity_group_member_instance_list( + pub async fn anti_affinity_group_member_list( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, - pagparams: &PaginatedBy<'_>, - ) -> ListResultVec { + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; - // TODO: Need to also look up "group_membership" here - // TODO: definitely test listing both? - // TODO: atlernately - make this API "instance member" specific, make a - // different one for "affinity group members", and paginate over both. - // - // That might be preferable. - - use db::schema::anti_affinity_group_instance_membership::dsl; - match pagparams { - PaginatedBy::Id(pagparams) => paginated( - dsl::anti_affinity_group_instance_membership, - dsl::instance_id, - &pagparams, - ), - PaginatedBy::Name(_) => { - return Err(Error::invalid_request( - "Cannot paginate group members by name", - )); + let mut query = QueryBuilder::new() + .sql( + " + SELECT instance_id as id, 'instance' as label + FROM anti_affinity_group_instance_membership + WHERE group_id = ", + ) + .param() + .bind::(authz_anti_affinity_group.id()) + .sql( + " + UNION + SELECT affinity_group_id as id, 'affinity_group' as label + FROM anti_affinity_group_affinity_membership + WHERE anti_affinity_group_id = ", + ) + .param() + .bind::(authz_anti_affinity_group.id()) + .sql(" "); + + let (sort, cmp) = match pagparams.direction { + dropshot::PaginationOrder::Ascending => (" ORDER BY id ASC ", ">"), + dropshot::PaginationOrder::Descending => { + (" ORDER BY id DESC ", "<") } - } - .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .select(AntiAffinityGroupInstanceMembership::as_select()) - .load_async(&*self.pool_connection_authorized(opctx).await?) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + }; + if let Some(id) = pagparams.marker { + query = query + .sql("WHERE id ") + .sql(cmp) + .sql(" ") + .param() + .bind::(*id); + }; + + query = query.sql(sort); + query = + query.sql(" LIMIT ").param().bind::( + i64::from(pagparams.limit.get()), + ); + + Ok(query + .query::<(diesel::sql_types::Uuid, diesel::sql_types::Text)>() + .load_async::<(Uuid, String)>( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|(id, label)| { + use external::AntiAffinityGroupMember as Member; + match label.as_str() { + "affinity_group" => Member::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(id), + ), + "instance" => { + Member::Instance(InstanceUuid::from_untyped_uuid(id)) + } + other => panic!("Unexpected label from query: {other}"), + } + }) + .collect()) } pub async fn affinity_group_member_view( @@ -1804,18 +1844,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -1843,11 +1878,7 @@ mod tests { // We should now be able to list the new member let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert_eq!(members.len(), 1); @@ -1870,11 +1901,7 @@ mod tests { .await .unwrap(); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2048,18 +2075,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2123,11 +2145,7 @@ mod tests { // We should now be able to list the new member let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert_eq!(members.len(), 1); @@ -2151,11 +2169,7 @@ mod tests { .await .unwrap(); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2266,18 +2280,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2309,11 +2318,7 @@ mod tests { // Confirm that no instance members exist let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2433,18 +2438,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2481,11 +2481,7 @@ mod tests { // Confirm that no instance members exist let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -3014,18 +3010,13 @@ mod tests { // // Two calls to "anti_affinity_group_member_add" should be the same // as a single call. - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert_eq!(members.len(), 1); @@ -3063,11 +3054,7 @@ mod tests { ); let members = datastore - .anti_affinity_group_member_instance_list( - &opctx, - &authz_group, - &pagbyid, - ) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 11c2fb10cea..bd19311fe94 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -15,6 +15,7 @@ use nexus_types::external_api::views; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; @@ -254,22 +255,18 @@ impl super::Nexus { &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, - pagparams: &PaginatedBy<'_>, + pagparams: &DataPageParams<'_, uuid::Uuid>, ) -> ListResultVec { let (.., authz_anti_affinity_group) = anti_affinity_group_lookup .lookup_for(authz::Action::ListChildren) .await?; - Ok(self - .db_datastore - .anti_affinity_group_member_instance_list( + self.db_datastore + .anti_affinity_group_member_list( opctx, &authz_anti_affinity_group, pagparams, ) - .await? - .into_iter() - .map(Into::into) - .collect()) + .await } pub(crate) async fn affinity_group_member_view( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index d9298f1af36..246cc86c551 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -2925,7 +2925,6 @@ impl NexusExternalApi for NexusExternalApiImpl { let query = query_params.into_inner(); let pag_params = data_page_params_for(&rqctx, &query)?; let scan_params = ScanById::from_query(&query)?; - let paginated_by = id_pagination(&pag_params, scan_params)?; let group_selector = params::AntiAffinityGroupSelector { project: scan_params.selector.project.clone(), @@ -2937,7 +2936,7 @@ impl NexusExternalApi for NexusExternalApiImpl { .anti_affinity_group_member_list( &opctx, &group_lookup, - &paginated_by, + &pag_params, ) .await?; Ok(HttpResponseOk(ScanById::results_page( From 0419d378d23c6efef6227708c0abf10582b7d24f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 24 Feb 2025 13:10:38 -0800 Subject: [PATCH 57/84] review feedback --- nexus/src/app/affinity.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index ca8d0c708cd..5a1ebf7ab22 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -53,7 +53,7 @@ impl super::Nexus { .. } => { Err(Error::invalid_request( - "when providing affinity_group as an ID project should not be specified", + "when providing affinity_group as an ID, project should not be specified", )) } _ => { @@ -92,7 +92,7 @@ impl super::Nexus { .. } => { Err(Error::invalid_request( - "when providing anti_affinity_group as an ID project should not be specified", + "when providing anti_affinity_group as an ID, project should not be specified", )) } _ => { @@ -190,7 +190,7 @@ impl super::Nexus { self.db_datastore .affinity_group_update(opctx, &authz_group, updates.clone().into()) .await - .map(|g| g.into()) + .map(Into::into) } pub(crate) async fn anti_affinity_group_update( @@ -208,7 +208,7 @@ impl super::Nexus { updates.clone().into(), ) .await - .map(|g| g.into()) + .map(Into::into) } pub(crate) async fn affinity_group_delete( From 36ddafc91a4ae418af4b24acd4496565e7a6bb60 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 24 Feb 2025 16:18:38 -0800 Subject: [PATCH 58/84] Relocate some functions into pub_test_utils --- nexus/db-queries/benches/harness/db_utils.rs | 289 +------------ nexus/db-queries/benches/harness/mod.rs | 3 +- nexus/db-queries/benches/sled_reservation.rs | 25 +- nexus/db-queries/src/db/datastore/affinity.rs | 267 +++--------- nexus/db-queries/src/db/datastore/mod.rs | 79 +--- .../src/db/datastore/physical_disk.rs | 22 +- nexus/db-queries/src/db/datastore/rack.rs | 23 +- nexus/db-queries/src/db/datastore/sled.rs | 199 +-------- nexus/db-queries/src/db/datastore/vpc.rs | 21 +- .../src/db/pub_test_utils/helpers.rs | 400 ++++++++++++++++++ nexus/db-queries/src/db/pub_test_utils/mod.rs | 1 + 11 files changed, 536 insertions(+), 793 deletions(-) create mode 100644 nexus/db-queries/src/db/pub_test_utils/helpers.rs diff --git a/nexus/db-queries/benches/harness/db_utils.rs b/nexus/db-queries/benches/harness/db_utils.rs index dad09beeaa0..fb778eb9a33 100644 --- a/nexus/db-queries/benches/harness/db_utils.rs +++ b/nexus/db-queries/benches/harness/db_utils.rs @@ -14,89 +14,29 @@ use anyhow::Context; use anyhow::Result; -use chrono::Utc; -use nexus_db_model::ByteCount; -use nexus_db_model::Generation; -use nexus_db_model::Instance; -use nexus_db_model::InstanceRuntimeState; -use nexus_db_model::InstanceState; -use nexus_db_model::Project; -use nexus_db_model::Resources; use nexus_db_model::Sled; -use nexus_db_model::SledBaseboard; use nexus_db_model::SledReservationConstraintBuilder; -use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; -use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; -use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::pub_test_utils::helpers::small_resource_request; +use nexus_db_queries::db::pub_test_utils::helpers::SledUpdateBuilder; use nexus_db_queries::db::DataStore; -use nexus_types::external_api::params; -use nexus_types::identity::Resource; -use omicron_common::api::external; -use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; -use std::net::Ipv6Addr; -use std::net::SocketAddrV6; -use std::str::FromStr; use uuid::Uuid; -pub async fn create_project( - opctx: &OpContext, - datastore: &DataStore, -) -> (authz::Project, Project) { - let authz_silo = opctx.authn.silo_required().unwrap(); - - // Create a project - let project = Project::new( - authz_silo.id(), - params::ProjectCreate { - identity: external::IdentityMetadataCreateParams { - name: "project".parse().unwrap(), - description: "desc".to_string(), - }, - }, - ); - datastore.project_create(&opctx, project).await.unwrap() -} - pub fn rack_id() -> Uuid { Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap() } -// Creates a "fake" Sled Baseboard. -pub fn sled_baseboard_for_test() -> SledBaseboard { - SledBaseboard { - serial_number: Uuid::new_v4().to_string(), - part_number: String::from("test-part"), - revision: 1, - } -} - -// Creates "fake" sled hardware accounting -pub fn sled_system_hardware_for_test() -> SledSystemHardware { - SledSystemHardware { - is_scrimlet: false, - usable_hardware_threads: 32, - usable_physical_ram: ByteCount::try_from(1 << 40).unwrap(), - reservoir_size: ByteCount::try_from(1 << 39).unwrap(), - } -} +const USABLE_HARDWARE_THREADS: u32 = 32; pub fn test_new_sled_update() -> SledUpdate { - let sled_id = Uuid::new_v4(); - let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); - let repo_depot_port = 0; - SledUpdate::new( - sled_id, - addr, - repo_depot_port, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id(), - Generation::new(), - ) + let mut sled = SledUpdateBuilder::new(); + sled.rack_id(rack_id()) + .hardware() + .usable_hardware_threads(USABLE_HARDWARE_THREADS); + sled.build() } pub async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { @@ -109,15 +49,6 @@ pub async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { sleds } -fn small_resource_request() -> Resources { - Resources::new( - 1, - // Just require the bare non-zero amount of RAM. - ByteCount::try_from(1024).unwrap(), - ByteCount::try_from(1024).unwrap(), - ) -} - /// Given a `sled_count`, returns the number of times a call to /// `create_reservation` should succeed. /// @@ -125,213 +56,11 @@ fn small_resource_request() -> Resources { pub fn max_resource_request_count(sled_count: usize) -> usize { let threads_per_request: usize = small_resource_request().hardware_threads.0.try_into().unwrap(); - let threads_per_sled: usize = sled_system_hardware_for_test() - .usable_hardware_threads - .try_into() - .unwrap(); + let threads_per_sled: usize = USABLE_HARDWARE_THREADS.try_into().unwrap(); threads_per_sled * sled_count / threads_per_request } -// Helper function for creating an instance without a VMM. -pub async fn create_instance_record( - opctx: &OpContext, - datastore: &DataStore, - authz_project: &authz::Project, - name: &str, -) -> InstanceUuid { - let instance = Instance::new( - InstanceUuid::new_v4(), - authz_project.id(), - ¶ms::InstanceCreate { - identity: external::IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "".to_string(), - }, - ncpus: 2i64.try_into().unwrap(), - memory: external::ByteCount::from_gibibytes_u32(16), - hostname: "myhostname".try_into().unwrap(), - user_data: Vec::new(), - network_interfaces: - params::InstanceNetworkInterfaceAttachment::None, - external_ips: Vec::new(), - disks: Vec::new(), - boot_disk: None, - ssh_public_keys: None, - start: false, - auto_restart_policy: Default::default(), - }, - ); - - let instance = datastore - .project_create_instance(&opctx, &authz_project, instance) - .await - .unwrap(); - - let id = InstanceUuid::from_untyped_uuid(instance.id()); - datastore - .instance_update_runtime( - &id, - &InstanceRuntimeState { - nexus_state: InstanceState::NoVmm, - time_updated: Utc::now(), - propolis_id: None, - migration_id: None, - dst_propolis_id: None, - gen: Generation::from(Generation::new().0.next()), - time_last_auto_restarted: None, - }, - ) - .await - .expect("Failed to update runtime state"); - - id -} -pub async fn delete_instance_record( - opctx: &OpContext, - datastore: &DataStore, - instance_id: InstanceUuid, -) { - let (.., authz_instance) = LookupPath::new(opctx, datastore) - .instance_id(instance_id.into_untyped_uuid()) - .lookup_for(authz::Action::Delete) - .await - .unwrap(); - datastore.project_delete_instance(&opctx, &authz_instance).await.unwrap(); -} - -pub async fn create_affinity_group( - opctx: &OpContext, - db: &DataStore, - authz_project: &authz::Project, - group_name: &'static str, - policy: external::AffinityPolicy, -) { - db.affinity_group_create( - &opctx, - &authz_project, - nexus_db_model::AffinityGroup::new( - authz_project.id(), - params::AffinityGroupCreate { - identity: external::IdentityMetadataCreateParams { - name: group_name.parse().unwrap(), - description: "desc".to_string(), - }, - policy, - failure_domain: external::FailureDomain::Sled, - }, - ), - ) - .await - .unwrap(); -} - -pub async fn delete_affinity_group( - opctx: &OpContext, - db: &DataStore, - group_name: &'static str, -) { - let project = external::Name::from_str("project").unwrap(); - let group = external::Name::from_str(group_name).unwrap(); - let (.., authz_group) = LookupPath::new(opctx, db) - .project_name_owned(project.into()) - .affinity_group_name_owned(group.into()) - .lookup_for(authz::Action::Delete) - .await - .unwrap(); - - db.affinity_group_delete(&opctx, &authz_group).await.unwrap(); -} - -pub async fn create_anti_affinity_group( - opctx: &OpContext, - db: &DataStore, - authz_project: &authz::Project, - group_name: &'static str, - policy: external::AffinityPolicy, -) { - db.anti_affinity_group_create( - &opctx, - &authz_project, - nexus_db_model::AntiAffinityGroup::new( - authz_project.id(), - params::AntiAffinityGroupCreate { - identity: external::IdentityMetadataCreateParams { - name: group_name.parse().unwrap(), - description: "desc".to_string(), - }, - policy, - failure_domain: external::FailureDomain::Sled, - }, - ), - ) - .await - .unwrap(); -} - -pub async fn delete_anti_affinity_group( - opctx: &OpContext, - db: &DataStore, - group_name: &'static str, -) { - let project = external::Name::from_str("project").unwrap(); - let group = external::Name::from_str(group_name).unwrap(); - let (.., authz_group) = LookupPath::new(opctx, db) - .project_name_owned(project.into()) - .anti_affinity_group_name_owned(group.into()) - .lookup_for(authz::Action::Delete) - .await - .unwrap(); - - db.anti_affinity_group_delete(&opctx, &authz_group).await.unwrap(); -} - -pub async fn create_affinity_group_member( - opctx: &OpContext, - db: &DataStore, - group_name: &'static str, - instance_id: InstanceUuid, -) -> Result<()> { - let project = external::Name::from_str("project").unwrap(); - let group = external::Name::from_str(group_name).unwrap(); - let (.., authz_group) = LookupPath::new(opctx, db) - .project_name_owned(project.into()) - .affinity_group_name_owned(group.into()) - .lookup_for(authz::Action::Modify) - .await?; - - db.affinity_group_member_add( - opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance_id), - ) - .await?; - Ok(()) -} - -pub async fn create_anti_affinity_group_member( - opctx: &OpContext, - db: &DataStore, - group_name: &'static str, - instance_id: InstanceUuid, -) -> Result<()> { - let project = external::Name::from_str("project").unwrap(); - let group = external::Name::from_str(group_name).unwrap(); - let (.., authz_group) = LookupPath::new(opctx, db) - .project_name_owned(project.into()) - .anti_affinity_group_name_owned(group.into()) - .lookup_for(authz::Action::Modify) - .await?; - - db.anti_affinity_group_member_add( - opctx, - &authz_group, - external::AntiAffinityGroupMember::Instance(instance_id), - ) - .await?; - Ok(()) -} - pub async fn create_reservation( opctx: &OpContext, db: &DataStore, diff --git a/nexus/db-queries/benches/harness/mod.rs b/nexus/db-queries/benches/harness/mod.rs index e4d8b1baf3a..6f7db092820 100644 --- a/nexus/db-queries/benches/harness/mod.rs +++ b/nexus/db-queries/benches/harness/mod.rs @@ -9,6 +9,7 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::pub_test_utils::helpers::create_project; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DataStore; use nexus_test_utils::sql::process_rows; @@ -86,7 +87,7 @@ impl TestHarness { let db = TestDatabase::new_with_datastore(log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; create_sleds(&datastore, sled_count).await; Self { db, authz_project } diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index f4ef4741029..eb54156699a 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -20,18 +20,18 @@ use tokio::sync::Barrier; mod harness; -use harness::db_utils::create_affinity_group; -use harness::db_utils::create_affinity_group_member; -use harness::db_utils::create_anti_affinity_group; -use harness::db_utils::create_anti_affinity_group_member; -use harness::db_utils::create_instance_record; use harness::db_utils::create_reservation; -use harness::db_utils::delete_affinity_group; -use harness::db_utils::delete_anti_affinity_group; -use harness::db_utils::delete_instance_record; use harness::db_utils::delete_reservation; use harness::db_utils::max_resource_request_count; use harness::TestHarness; +use nexus_db_queries::db::pub_test_utils::helpers::create_affinity_group; +use nexus_db_queries::db::pub_test_utils::helpers::create_affinity_group_member; +use nexus_db_queries::db::pub_test_utils::helpers::create_anti_affinity_group; +use nexus_db_queries::db::pub_test_utils::helpers::create_anti_affinity_group_member; +use nexus_db_queries::db::pub_test_utils::helpers::create_stopped_instance_record; +use nexus_db_queries::db::pub_test_utils::helpers::delete_affinity_group; +use nexus_db_queries::db::pub_test_utils::helpers::delete_anti_affinity_group; +use nexus_db_queries::db::pub_test_utils::helpers::delete_instance_record; ///////////////////////////////////////////////////////////////// // @@ -130,7 +130,7 @@ async fn create_instances_with_groups( ) -> Vec { let mut instance_ids = Vec::with_capacity(params.vmms); for i in 0..params.vmms { - let instance_id = create_instance_record( + let instance_id = create_stopped_instance_record( opctx, db, authz_project, @@ -149,6 +149,7 @@ async fn create_instances_with_groups( create_affinity_group_member( opctx, db, + "project", group.name, instance_id, ) @@ -159,6 +160,7 @@ async fn create_instances_with_groups( create_anti_affinity_group_member( opctx, db, + "project", group.name, instance_id, ) @@ -297,10 +299,11 @@ async fn delete_test_groups( for group in all_groups { match group.flavor { GroupType::Affinity => { - delete_affinity_group(opctx, db, group.name).await; + delete_affinity_group(opctx, db, "project", group.name).await; } GroupType::AntiAffinity => { - delete_anti_affinity_group(opctx, db, group.name).await; + delete_anti_affinity_group(opctx, db, "project", group.name) + .await; } } } diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 3fe3660e7dd..183c9caad30 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -875,8 +875,9 @@ mod tests { use super::*; use crate::db::lookup::LookupPath; + use crate::db::pub_test_utils::helpers::create_project; + use crate::db::pub_test_utils::helpers::create_stopped_instance_record; use crate::db::pub_test_utils::TestDatabase; - use nexus_db_model::Instance; use nexus_db_model::Resources; use nexus_db_model::SledResourceVmm; use nexus_types::external_api::params; @@ -884,30 +885,12 @@ mod tests { self, ByteCount, DataPageParams, IdentityMetadataCreateParams, }; use omicron_test_utils::dev; + use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; use std::num::NonZeroU32; - // Helper function for creating a project - async fn create_project( - opctx: &OpContext, - datastore: &DataStore, - name: &str, - ) -> (authz::Project, Project) { - let authz_silo = opctx.authn.silo_required().unwrap(); - let project = Project::new( - authz_silo.id(), - params::ProjectCreate { - identity: IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "".to_string(), - }, - }, - ); - datastore.project_create(&opctx, project).await.unwrap() - } - // Helper function for creating an affinity group with // arbitrary configuration. async fn create_affinity_group( @@ -954,64 +937,6 @@ mod tests { .await } - // Helper function for creating an instance without a VMM. - async fn create_instance_record( - opctx: &OpContext, - datastore: &DataStore, - authz_project: &authz::Project, - name: &str, - ) -> Instance { - let instance = Instance::new( - InstanceUuid::new_v4(), - authz_project.id(), - ¶ms::InstanceCreate { - identity: IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "".to_string(), - }, - ncpus: 2i64.try_into().unwrap(), - memory: ByteCount::from_gibibytes_u32(16), - hostname: "myhostname".try_into().unwrap(), - user_data: Vec::new(), - network_interfaces: - params::InstanceNetworkInterfaceAttachment::None, - external_ips: Vec::new(), - disks: Vec::new(), - boot_disk: None, - ssh_public_keys: None, - start: false, - auto_restart_policy: Default::default(), - }, - ); - - let instance = datastore - .project_create_instance(&opctx, &authz_project, instance) - .await - .unwrap(); - - set_instance_state_stopped(&datastore, instance.id()).await; - - instance - } - - async fn set_instance_state_stopped( - datastore: &DataStore, - instance: uuid::Uuid, - ) { - use db::schema::instance::dsl; - diesel::update(dsl::instance) - .filter(dsl::id.eq(instance)) - .set(( - dsl::state.eq(db::model::InstanceState::NoVmm), - dsl::active_propolis_id.eq(None::), - )) - .execute_async( - &*datastore.pool_connection_for_tests().await.unwrap(), - ) - .await - .unwrap(); - } - // Helper for explicitly modifying sled resource usage // // The interaction we typically use to create and modify instance state @@ -1480,7 +1405,7 @@ mod tests { assert!(members.is_empty()); // Create an instance without a VMM. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -1493,9 +1418,7 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -1507,9 +1430,7 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()) - ), + external::AffinityGroupMember::Instance(instance,), members[0].clone().into() ); @@ -1518,9 +1439,7 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -1576,7 +1495,7 @@ mod tests { assert!(members.is_empty()); // Create an instance without a VMM. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -1589,9 +1508,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -1603,9 +1520,7 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()) - ), + external::AntiAffinityGroupMember::Instance(instance,), members[0].clone().into() ); @@ -1614,9 +1529,7 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -1672,7 +1585,7 @@ mod tests { assert!(members.is_empty()); // Create an instance with a VMM. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -1680,20 +1593,14 @@ mod tests { ) .await; - allocate_instance_reservation( - &datastore, - InstanceUuid::from_untyped_uuid(instance.id()), - ) - .await; + allocate_instance_reservation(&datastore, instance).await; // Cannot add the instance to the group while it's running. let err = datastore .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .expect_err( @@ -1708,28 +1615,18 @@ mod tests { ); // If we have no reservation for the instance, we can add it to the group. - delete_instance_reservation( - &datastore, - InstanceUuid::from_untyped_uuid(instance.id()), - ) - .await; + delete_instance_reservation(&datastore, instance).await; datastore .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); // Now we can reserve a sled for the instance once more. - allocate_instance_reservation( - &datastore, - InstanceUuid::from_untyped_uuid(instance.id()), - ) - .await; + allocate_instance_reservation(&datastore, instance).await; // We should now be able to list the new member let members = datastore @@ -1738,9 +1635,7 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()) - ), + external::AffinityGroupMember::Instance(instance,), members[0].clone().into() ); @@ -1750,9 +1645,7 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -1808,25 +1701,21 @@ mod tests { assert!(members.is_empty()); // Create an instance with a VMM. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, "my-instance", ) .await; - allocate_instance_reservation( - &datastore, - InstanceUuid::from_untyped_uuid(instance.id()), - ) - .await; + allocate_instance_reservation(&datastore, instance).await; // Cannot add the instance to the group while it's running. let err = datastore .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(InstanceUuid::from_untyped_uuid(instance.id())), + external::AntiAffinityGroupMember::Instance(instance), ) .await .expect_err( @@ -1841,28 +1730,18 @@ mod tests { ); // If we have no reservation for the instance, we can add it to the group. - delete_instance_reservation( - &datastore, - InstanceUuid::from_untyped_uuid(instance.id()), - ) - .await; + delete_instance_reservation(&datastore, instance).await; datastore .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); // Now we can reserve a sled for the instance once more. - allocate_instance_reservation( - &datastore, - InstanceUuid::from_untyped_uuid(instance.id()), - ) - .await; + allocate_instance_reservation(&datastore, instance).await; // We should now be able to list the new member let members = datastore @@ -1871,9 +1750,7 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()) - ), + external::AntiAffinityGroupMember::Instance(instance,), members[0].clone().into() ); @@ -1883,9 +1760,7 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -1940,7 +1815,7 @@ mod tests { assert!(members.is_empty()); // Create an instance without a VMM, add it to the group. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -1951,9 +1826,7 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -2014,7 +1887,7 @@ mod tests { assert!(members.is_empty()); // Create an instance without a VMM, add it to the group. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2025,9 +1898,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -2091,7 +1962,7 @@ mod tests { assert!(members.is_empty()); // Create an instance without a VMM, add it to the group. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2102,16 +1973,14 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); // Delete the instance let (.., authz_instance) = LookupPath::new(opctx, datastore) - .instance_id(instance.id()) + .instance_id(instance.into_untyped_uuid()) .lookup_for(authz::Action::Delete) .await .unwrap(); @@ -2173,7 +2042,7 @@ mod tests { assert!(members.is_empty()); // Create an instance without a VMM, add it to the group. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2184,16 +2053,14 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); // Delete the instance let (.., authz_instance) = LookupPath::new(opctx, datastore) - .instance_id(instance.id()) + .instance_id(instance.into_untyped_uuid()) .lookup_for(authz::Action::Delete) .await .unwrap(); @@ -2265,7 +2132,7 @@ mod tests { } // Create an instance, and maybe delete it. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2273,7 +2140,7 @@ mod tests { ) .await; let (.., authz_instance) = LookupPath::new(opctx, datastore) - .instance_id(instance.id()) + .instance_id(instance.into_untyped_uuid()) .lookup_for(authz::Action::Modify) .await .unwrap(); @@ -2292,9 +2159,7 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .expect_err("Should have failed"); @@ -2326,9 +2191,7 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .expect_err("Should have failed"); @@ -2425,7 +2288,7 @@ mod tests { } // Create an instance, and maybe delete it. - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2433,7 +2296,7 @@ mod tests { ) .await; let (.., authz_instance) = LookupPath::new(opctx, datastore) - .instance_id(instance.id()) + .instance_id(instance.into_untyped_uuid()) .lookup_for(authz::Action::Modify) .await .unwrap(); @@ -2452,9 +2315,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .expect_err("Should have failed"); @@ -2486,9 +2347,7 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .expect_err("Should have failed"); @@ -2560,7 +2419,7 @@ mod tests { .unwrap(); // Create an instance - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2573,9 +2432,7 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -2585,9 +2442,7 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap_err(); @@ -2623,9 +2478,7 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -2633,9 +2486,7 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AffinityGroupMember::Instance(instance), ) .await .unwrap_err(); @@ -2687,7 +2538,7 @@ mod tests { .unwrap(); // Create an instance - let instance = create_instance_record( + let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, @@ -2700,9 +2551,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -2712,9 +2561,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap_err(); @@ -2750,9 +2597,7 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); @@ -2760,9 +2605,7 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(instance.id()), - ), + external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap_err(); diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 0e6a939581f..6740adf6824 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -460,9 +460,9 @@ mod test { use crate::db::model::{ BlockSize, ConsoleSession, CrucibleDataset, ExternalIp, PhysicalDisk, PhysicalDiskKind, PhysicalDiskPolicy, PhysicalDiskState, Project, Rack, - Region, SiloUser, SledBaseboard, SledSystemHardware, SledUpdate, - SshKey, Zpool, + Region, SiloUser, SshKey, Zpool, }; + use crate::db::pub_test_utils::helpers::SledUpdateBuilder; use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::vpc_subnet::InsertVpcSubnetQuery; use chrono::{Duration, Utc}; @@ -470,8 +470,8 @@ mod test { use futures::StreamExt; use nexus_config::RegionAllocationStrategy; use nexus_db_fixed_data::silo::DEFAULT_SILO; + use nexus_db_model::to_db_typed_uuid; use nexus_db_model::IpAttachState; - use nexus_db_model::{to_db_typed_uuid, Generation}; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintTarget; use nexus_types::external_api::params; @@ -494,27 +494,6 @@ mod test { use strum::EnumCount; use uuid::Uuid; - // Creates a "fake" Sled Baseboard. - pub fn sled_baseboard_for_test() -> SledBaseboard { - SledBaseboard { - serial_number: Uuid::new_v4().to_string(), - part_number: String::from("test-part"), - revision: 1, - } - } - - // Creates "fake" sled hardware accounting - pub fn sled_system_hardware_for_test() -> SledSystemHardware { - SledSystemHardware { - is_scrimlet: false, - usable_hardware_threads: 4, - usable_physical_ram: crate::db::model::ByteCount::try_from(1 << 40) - .unwrap(), - reservoir_size: crate::db::model::ByteCount::try_from(1 << 39) - .unwrap(), - } - } - /// Inserts a blueprint in the DB and forcibly makes it the target /// /// WARNING: This makes no attempts to validate the blueprint relative to @@ -716,25 +695,8 @@ mod test { // Creates a test sled, returns its UUID. async fn create_test_sled(datastore: &DataStore) -> SledUuid { - let bogus_addr = SocketAddrV6::new( - Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), - 8080, - 0, - 0, - ); - let bogus_repo_depot_port = 8081; - let rack_id = Uuid::new_v4(); let sled_id = SledUuid::new_v4(); - - let sled_update = SledUpdate::new( - sled_id.into_untyped_uuid(), - bogus_addr, - bogus_repo_depot_port, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id, - Generation::new(), - ); + let sled_update = SledUpdateBuilder::new().sled_id(sled_id).build(); datastore.sled_upsert(sled_update).await.unwrap(); sled_id } @@ -1716,32 +1678,25 @@ mod test { let rack_id = Uuid::new_v4(); let addr1 = "[fd00:1de::1]:12345".parse().unwrap(); let sled1_id = "0de4b299-e0b4-46f0-d528-85de81a7095f".parse().unwrap(); - let sled1 = db::model::SledUpdate::new( - sled1_id, - addr1, - REPO_DEPOT_PORT, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id, - Generation::new(), - ); + + let sled1 = SledUpdateBuilder::new() + .sled_id(sled1_id) + .addr(addr1) + .repo_depot_port(REPO_DEPOT_PORT) + .rack_id(rack_id) + .build(); datastore.sled_upsert(sled1).await.unwrap(); let addr2 = "[fd00:1df::1]:12345".parse().unwrap(); let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap(); - let sled2 = db::model::SledUpdate::new( - sled2_id, - addr2, - REPO_DEPOT_PORT, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id, - Generation::new(), - ); + let sled2 = SledUpdateBuilder::new() + .sled_id(sled2_id) + .addr(addr2) + .repo_depot_port(REPO_DEPOT_PORT) + .rack_id(rack_id) + .build(); datastore.sled_upsert(sled2).await.unwrap(); - let sled1_id = SledUuid::from_untyped_uuid(sled1_id); - let sled2_id = SledUuid::from_untyped_uuid(sled2_id); let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap(); let expected_ip = Ipv6Addr::new(0xfd00, 0x1de, 0, 0, 0, 0, 1, 0); assert_eq!(ip, expected_ip); diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs index 796ee27ca9f..94d094c9889 100644 --- a/nexus/db-queries/src/db/datastore/physical_disk.rs +++ b/nexus/db-queries/src/db/datastore/physical_disk.rs @@ -333,14 +333,11 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::test::{ - sled_baseboard_for_test, sled_system_hardware_for_test, - }; use crate::db::lookup::LookupPath; - use crate::db::model::{PhysicalDiskKind, Sled, SledUpdate}; + use crate::db::model::{PhysicalDiskKind, Sled}; + use crate::db::pub_test_utils::helpers::SledUpdateBuilder; use crate::db::pub_test_utils::TestDatabase; use dropshot::PaginationOrder; - use nexus_db_model::Generation; use nexus_sled_agent_shared::inventory::{ Baseboard, Inventory, InventoryDisk, OmicronZonesConfig, SledRole, }; @@ -348,23 +345,10 @@ mod test { use omicron_common::api::external::ByteCount; use omicron_common::disk::{DiskIdentity, DiskVariant}; use omicron_test_utils::dev; - use std::net::{Ipv6Addr, SocketAddrV6}; use std::num::NonZeroU32; async fn create_test_sled(db: &DataStore) -> Sled { - let sled_id = Uuid::new_v4(); - let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); - let repo_depot_port = 0; - let rack_id = Uuid::new_v4(); - let sled_update = SledUpdate::new( - sled_id, - addr, - repo_depot_port, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id, - Generation::new(), - ); + let sled_update = SledUpdateBuilder::new().build(); let (sled, _) = db .sled_upsert(sled_update) .await diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index d2dd84063ae..c83361391df 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -994,19 +994,17 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::test::{ - sled_baseboard_for_test, sled_system_hardware_for_test, - }; use crate::db::datastore::Discoverability; use crate::db::model::ExternalIp; use crate::db::model::IpKind; use crate::db::model::IpPoolRange; use crate::db::model::Sled; + use crate::db::pub_test_utils::helpers::SledUpdateBuilder; use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::AsyncSimpleConnection; use internal_dns_types::names::DNS_ZONE; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; - use nexus_db_model::{DnsGroup, Generation, InitialDnsGroup, SledUpdate}; + use nexus_db_model::{DnsGroup, Generation, InitialDnsGroup}; use nexus_inventory::now_db_precision; use nexus_reconfigurator_planning::system::{ SledBuilder, SystemDescription, @@ -1042,7 +1040,7 @@ mod test { use omicron_uuid_kinds::{SledUuid, TypedUuid}; use oxnet::IpNet; use std::collections::{BTreeMap, HashMap}; - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::num::NonZeroU32; // Default impl is for tests only, and really just so that tests can more @@ -1235,17 +1233,10 @@ mod test { } async fn create_test_sled(db: &DataStore, sled_id: Uuid) -> Sled { - let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); - let repo_depot_port = 0; - let sled_update = SledUpdate::new( - sled_id, - addr, - repo_depot_port, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id(), - Generation::new(), - ); + let sled_update = SledUpdateBuilder::new() + .sled_id(SledUuid::from_untyped_uuid(sled_id)) + .rack_id(rack_id()) + .build(); let (sled, _) = db .sled_upsert(sled_update) .await diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 4201bf7bc51..c7298b5b941 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -1111,19 +1111,18 @@ impl TransitionError { #[cfg(test)] pub(in crate::db::datastore) mod test { use super::*; - use crate::db::datastore::test::{ - sled_baseboard_for_test, sled_system_hardware_for_test, - }; use crate::db::datastore::test_utils::{ sled_set_policy, sled_set_state, Expected, IneligibleSleds, }; use crate::db::lookup::LookupPath; use crate::db::model::to_db_typed_uuid; - use crate::db::model::AffinityGroup; - use crate::db::model::AntiAffinityGroup; use crate::db::model::ByteCount; - use crate::db::model::Project; use crate::db::model::SqlU32; + use crate::db::pub_test_utils::helpers::create_affinity_group; + use crate::db::pub_test_utils::helpers::create_anti_affinity_group; + use crate::db::pub_test_utils::helpers::create_project; + use crate::db::pub_test_utils::helpers::small_resource_request; + use crate::db::pub_test_utils::helpers::SledUpdateBuilder; use crate::db::pub_test_utils::TestDatabase; use anyhow::{Context, Result}; use itertools::Itertools; @@ -1132,7 +1131,6 @@ pub(in crate::db::datastore) mod test { use nexus_db_model::PhysicalDiskKind; use nexus_db_model::PhysicalDiskPolicy; use nexus_db_model::PhysicalDiskState; - use nexus_types::external_api::params; use nexus_types::identity::Asset; use nexus_types::identity::Resource; use omicron_common::api::external; @@ -1144,7 +1142,6 @@ pub(in crate::db::datastore) mod test { use omicron_uuid_kinds::SledUuid; use predicates::{prelude::*, BoxPredicate}; use std::collections::HashMap; - use std::net::{Ipv6Addr, SocketAddrV6}; fn rack_id() -> Uuid { Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap() @@ -1156,66 +1153,6 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let datastore = db.datastore(); - let mut sled_update = test_new_sled_update(); - let (observed_sled, _) = - datastore.sled_upsert(sled_update.clone()).await.unwrap(); - assert_eq!( - observed_sled.usable_hardware_threads, - sled_update.usable_hardware_threads - ); - assert_eq!( - observed_sled.usable_physical_ram, - sled_update.usable_physical_ram - ); - assert_eq!(observed_sled.reservoir_size, sled_update.reservoir_size); - - // Modify the sizes of hardware - sled_update.usable_hardware_threads = - SqlU32::new(sled_update.usable_hardware_threads.0 + 1); - const MIB: u64 = 1024 * 1024; - sled_update.usable_physical_ram = ByteCount::from( - external::ByteCount::try_from( - sled_update.usable_physical_ram.0.to_bytes() + MIB, - ) - .unwrap(), - ); - sled_update.reservoir_size = ByteCount::from( - external::ByteCount::try_from( - sled_update.reservoir_size.0.to_bytes() + MIB, - ) - .unwrap(), - ); - - // Bump the generation number so the insert succeeds. - sled_update.sled_agent_gen.0 = sled_update.sled_agent_gen.0.next(); - - // Test that upserting the sled propagates those changes to the DB. - let (observed_sled, _) = datastore - .sled_upsert(sled_update.clone()) - .await - .expect("Could not upsert sled during test prep"); - assert_eq!( - observed_sled.usable_hardware_threads, - sled_update.usable_hardware_threads - ); - assert_eq!( - observed_sled.usable_physical_ram, - sled_update.usable_physical_ram - ); - assert_eq!(observed_sled.reservoir_size, sled_update.reservoir_size); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn upsert_sled_updates_fails_with_stale_sled_agent_gen() { - let logctx = dev::test_setup_log( - "upsert_sled_updates_fails_with_stale_sled_agent_gen", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let datastore = db.datastore(); - let mut sled_update = test_new_sled_update(); let (observed_sled, _) = datastore.sled_upsert(sled_update.clone()).await.unwrap(); @@ -1439,71 +1376,17 @@ pub(in crate::db::datastore) mod test { // Utilities to help with Affinity Testing - // Create a resource request that will probably fit on a sled. - fn small_resource_request() -> db::model::Resources { - db::model::Resources::new( - 1, - // Just require the bare non-zero amount of RAM. - ByteCount::try_from(1024).unwrap(), - ByteCount::try_from(1024).unwrap(), - ) - } - // Create a resource request that will entirely fill a sled. fn large_resource_request() -> db::model::Resources { - let sled_resources = sled_system_hardware_for_test(); + // NOTE: This is dependent on [`test_new_sled_update`] using the default + // configuration for [`SledSystemHardware`]. + let sled_resources = SledUpdateBuilder::new().hardware().build(); let threads = sled_resources.usable_hardware_threads; let rss_ram = sled_resources.usable_physical_ram; let reservoir_ram = sled_resources.reservoir_size; db::model::Resources::new(threads, rss_ram, reservoir_ram) } - async fn create_project( - opctx: &OpContext, - datastore: &DataStore, - ) -> (authz::Project, db::model::Project) { - let authz_silo = opctx.authn.silo_required().unwrap(); - - // Create a project - let project = Project::new( - authz_silo.id(), - params::ProjectCreate { - identity: external::IdentityMetadataCreateParams { - name: "project".parse().unwrap(), - description: "desc".to_string(), - }, - }, - ); - datastore.project_create(&opctx, project).await.unwrap() - } - - async fn create_anti_affinity_group( - opctx: &OpContext, - datastore: &DataStore, - authz_project: &authz::Project, - name: &'static str, - policy: external::AffinityPolicy, - ) -> AntiAffinityGroup { - datastore - .anti_affinity_group_create( - &opctx, - &authz_project, - AntiAffinityGroup::new( - authz_project.id(), - params::AntiAffinityGroupCreate { - identity: external::IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "desc".to_string(), - }, - policy, - failure_domain: external::FailureDomain::Sled, - }, - ), - ) - .await - .unwrap() - } - // This short-circuits some of the logic and checks we normally have when // creating affinity groups, but makes testing easier. async fn add_instance_to_anti_affinity_group( @@ -1525,33 +1408,6 @@ pub(in crate::db::datastore) mod test { .unwrap(); } - async fn create_affinity_group( - opctx: &OpContext, - datastore: &DataStore, - authz_project: &authz::Project, - name: &'static str, - policy: external::AffinityPolicy, - ) -> AffinityGroup { - datastore - .affinity_group_create( - &opctx, - &authz_project, - AffinityGroup::new( - authz_project.id(), - params::AffinityGroupCreate { - identity: external::IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: "desc".to_string(), - }, - policy, - failure_domain: external::FailureDomain::Sled, - }, - ), - ) - .await - .unwrap() - } - // This short-circuits some of the logic and checks we normally have when // creating affinity groups, but makes testing easier. async fn add_instance_to_affinity_group( @@ -1776,7 +1632,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -1825,7 +1681,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 3; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -1871,7 +1727,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -1919,7 +1775,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -1966,7 +1822,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 3; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2029,7 +1885,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2082,7 +1938,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2130,7 +1986,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2189,7 +2045,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 2; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2248,7 +2104,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2316,7 +2172,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2384,7 +2240,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 3; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2443,7 +2299,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 3; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2824,18 +2680,7 @@ pub(in crate::db::datastore) mod test { // --- pub(crate) fn test_new_sled_update() -> SledUpdate { - let sled_id = Uuid::new_v4(); - let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); - let repo_depot_port = 0; - SledUpdate::new( - sled_id, - addr, - repo_depot_port, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id(), - Generation::new(), - ) + SledUpdateBuilder::new().rack_id(rack_id()).build() } /// Initial state for state transitions. diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index ba34f2d9ad5..b4fb2c8faa5 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2867,16 +2867,14 @@ fn is_services_vpc_gateway(igw: &InternetGateway) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::test::sled_baseboard_for_test; - use crate::db::datastore::test::sled_system_hardware_for_test; use crate::db::datastore::test_utils::IneligibleSleds; use crate::db::model::Project; + use crate::db::pub_test_utils::helpers::SledUpdateBuilder; use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; use nexus_db_fixed_data::silo::DEFAULT_SILO; use nexus_db_fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use nexus_db_model::IncompleteNetworkInterface; - use nexus_db_model::SledUpdate; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; use nexus_reconfigurator_planning::system::SledBuilder; use nexus_reconfigurator_planning::system::SystemDescription; @@ -3167,18 +3165,11 @@ mod tests { let sled_id = SledUuid::new_v4(); sled_ids.push(sled_id); system.sled(SledBuilder::new().id(sled_id)).expect("adding sled"); - datastore - .sled_upsert(SledUpdate::new( - sled_id.into_untyped_uuid(), - "[::1]:0".parse().unwrap(), - 0, - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - rack_id, - Generation::new().into(), - )) - .await - .expect("upserting sled"); + let sled_update = SledUpdateBuilder::new() + .sled_id(sled_id) + .rack_id(rack_id) + .build(); + datastore.sled_upsert(sled_update).await.expect("upserting sled"); } sled_ids.sort_unstable(); let planning_input = system diff --git a/nexus/db-queries/src/db/pub_test_utils/helpers.rs b/nexus/db-queries/src/db/pub_test_utils/helpers.rs new file mode 100644 index 00000000000..88dc9c3047b --- /dev/null +++ b/nexus/db-queries/src/db/pub_test_utils/helpers.rs @@ -0,0 +1,400 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database test helpers. These are wrappers to make testing more compact. + +use crate::authz; +use crate::context::OpContext; +use crate::db::lookup::LookupPath; +use crate::db::DataStore; + +use anyhow::Result; +use chrono::Utc; +use nexus_db_model::AffinityGroup; +use nexus_db_model::AntiAffinityGroup; +use nexus_db_model::ByteCount; +use nexus_db_model::Generation; +use nexus_db_model::Instance; +use nexus_db_model::InstanceRuntimeState; +use nexus_db_model::InstanceState; +use nexus_db_model::Project; +use nexus_db_model::Resources; +use nexus_db_model::SledBaseboard; +use nexus_db_model::SledSystemHardware; +use nexus_db_model::SledUpdate; +use nexus_types::external_api::params; +use nexus_types::identity::Resource; +use omicron_common::api::external; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::SledUuid; +use std::net::Ipv6Addr; +use std::net::SocketAddrV6; +use std::str::FromStr; +use uuid::Uuid; + +/// Creates a project within the silo of "opctx". +pub async fn create_project( + opctx: &OpContext, + datastore: &DataStore, + name: &'static str, +) -> (authz::Project, Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() +} + +/// Creates a "fake" [`SledBaseboard`] +pub fn sled_baseboard_for_test() -> SledBaseboard { + SledBaseboard { + serial_number: Uuid::new_v4().to_string(), + part_number: String::from("test-part"), + revision: 1, + } +} + +/// A utility for creating a [`SledSystemHardware`] +pub struct SledSystemHardwareBuilder { + is_scrimlet: bool, + usable_hardware_threads: u32, + usable_physical_ram: i64, + reservoir_size: i64, +} + +impl Default for SledSystemHardwareBuilder { + fn default() -> Self { + Self { + is_scrimlet: false, + usable_hardware_threads: 4, + usable_physical_ram: 1 << 40, + reservoir_size: 1 << 39, + } + } +} + +impl SledSystemHardwareBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn is_scrimlet(&mut self, is_scrimlet: bool) -> &mut Self { + self.is_scrimlet = is_scrimlet; + self + } + + pub fn usable_hardware_threads( + &mut self, + usable_hardware_threads: u32, + ) -> &mut Self { + self.usable_hardware_threads = usable_hardware_threads; + self + } + + pub fn usable_physical_ram( + &mut self, + usable_physical_ram: i64, + ) -> &mut Self { + self.usable_physical_ram = usable_physical_ram; + self + } + + pub fn reservoir_size(&mut self, reservoir_size: i64) -> &mut Self { + self.reservoir_size = reservoir_size; + self + } + + pub fn build(&self) -> SledSystemHardware { + SledSystemHardware { + is_scrimlet: self.is_scrimlet, + usable_hardware_threads: self.usable_hardware_threads, + usable_physical_ram: self.usable_physical_ram.try_into().unwrap(), + reservoir_size: self.reservoir_size.try_into().unwrap(), + } + } +} + +/// A utility for creating a [`SledUpdate`]. +pub struct SledUpdateBuilder { + sled_id: SledUuid, + addr: SocketAddrV6, + repo_depot_port: u16, + rack_id: Uuid, + sled_hardware: SledSystemHardwareBuilder, +} + +impl Default for SledUpdateBuilder { + fn default() -> Self { + Self { + sled_id: SledUuid::new_v4(), + addr: SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0), + repo_depot_port: 0, + rack_id: Uuid::new_v4(), + sled_hardware: SledSystemHardwareBuilder::default(), + } + } +} + +impl SledUpdateBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn sled_id(&mut self, sled_id: SledUuid) -> &mut Self { + self.sled_id = sled_id; + self + } + + pub fn addr(&mut self, addr: SocketAddrV6) -> &mut Self { + self.addr = addr; + self + } + + pub fn repo_depot_port(&mut self, repo_depot_port: u16) -> &mut Self { + self.repo_depot_port = repo_depot_port; + self + } + + pub fn rack_id(&mut self, rack_id: Uuid) -> &mut Self { + self.rack_id = rack_id; + self + } + + pub fn hardware(&mut self) -> &mut SledSystemHardwareBuilder { + &mut self.sled_hardware + } + + pub fn build(&self) -> SledUpdate { + SledUpdate::new( + self.sled_id.into_untyped_uuid(), + self.addr, + self.repo_depot_port, + sled_baseboard_for_test(), + self.sled_hardware.build(), + self.rack_id, + Generation::new(), + ) + } +} + +pub fn small_resource_request() -> Resources { + Resources::new( + 1, + // Just require the bare non-zero amount of RAM. + ByteCount::try_from(1024).unwrap(), + ByteCount::try_from(1024).unwrap(), + ) +} + +/// Helper function for creating an instance without a VMM. +pub async fn create_stopped_instance_record( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, +) -> InstanceUuid { + let instance = Instance::new( + InstanceUuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + ncpus: 2i64.try_into().unwrap(), + memory: external::ByteCount::from_gibibytes_u32(16), + hostname: "myhostname".try_into().unwrap(), + user_data: Vec::new(), + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: Vec::new(), + disks: Vec::new(), + boot_disk: None, + ssh_public_keys: None, + start: false, + auto_restart_policy: Default::default(), + }, + ); + + let instance = datastore + .project_create_instance(&opctx, &authz_project, instance) + .await + .unwrap(); + + let id = InstanceUuid::from_untyped_uuid(instance.id()); + datastore + .instance_update_runtime( + &id, + &InstanceRuntimeState { + nexus_state: InstanceState::NoVmm, + time_updated: Utc::now(), + propolis_id: None, + migration_id: None, + dst_propolis_id: None, + gen: Generation::from(Generation::new().0.next()), + time_last_auto_restarted: None, + }, + ) + .await + .expect("Failed to update runtime state"); + + id +} + +pub async fn delete_instance_record( + opctx: &OpContext, + datastore: &DataStore, + instance_id: InstanceUuid, +) { + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance_id.into_untyped_uuid()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.project_delete_instance(&opctx, &authz_instance).await.unwrap(); +} + +pub async fn create_affinity_group( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + group_name: &'static str, + policy: external::AffinityPolicy, +) -> AffinityGroup { + db.affinity_group_create( + &opctx, + &authz_project, + nexus_db_model::AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap() +} + +pub async fn delete_affinity_group( + opctx: &OpContext, + db: &DataStore, + project_name: &'static str, + group_name: &'static str, +) { + let project = external::Name::from_str(project_name).unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + + db.affinity_group_delete(&opctx, &authz_group).await.unwrap(); +} + +pub async fn create_anti_affinity_group( + opctx: &OpContext, + db: &DataStore, + authz_project: &authz::Project, + group_name: &'static str, + policy: external::AffinityPolicy, +) -> AntiAffinityGroup { + db.anti_affinity_group_create( + &opctx, + &authz_project, + nexus_db_model::AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap() +} + +pub async fn delete_anti_affinity_group( + opctx: &OpContext, + db: &DataStore, + project_name: &'static str, + group_name: &'static str, +) { + let project = external::Name::from_str(project_name).unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .anti_affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + + db.anti_affinity_group_delete(&opctx, &authz_group).await.unwrap(); +} + +pub async fn create_affinity_group_member( + opctx: &OpContext, + db: &DataStore, + project_name: &'static str, + group_name: &'static str, + instance_id: InstanceUuid, +) -> Result<()> { + let project = external::Name::from_str(project_name).unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Modify) + .await?; + + db.affinity_group_member_add( + opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance_id), + ) + .await?; + Ok(()) +} + +pub async fn create_anti_affinity_group_member( + opctx: &OpContext, + db: &DataStore, + project_name: &'static str, + group_name: &'static str, + instance_id: InstanceUuid, +) -> Result<()> { + let project = external::Name::from_str(project_name).unwrap(); + let group = external::Name::from_str(group_name).unwrap(); + let (.., authz_group) = LookupPath::new(opctx, db) + .project_name_owned(project.into()) + .anti_affinity_group_name_owned(group.into()) + .lookup_for(authz::Action::Modify) + .await?; + + db.anti_affinity_group_member_add( + opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance_id), + ) + .await?; + Ok(()) +} diff --git a/nexus/db-queries/src/db/pub_test_utils/mod.rs b/nexus/db-queries/src/db/pub_test_utils/mod.rs index 3e184538030..84017be6454 100644 --- a/nexus/db-queries/src/db/pub_test_utils/mod.rs +++ b/nexus/db-queries/src/db/pub_test_utils/mod.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use uuid::Uuid; pub mod crdb; +pub mod helpers; enum Populate { Nothing, From b8c5024c067acf13bd585cf2f0e7d04e237e9ab7 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 24 Feb 2025 16:36:59 -0800 Subject: [PATCH 59/84] Code review feedback --- nexus/db-queries/benches/harness/mod.rs | 23 +++++++++++++++++--- nexus/db-queries/benches/sled_reservation.rs | 15 ++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/nexus/db-queries/benches/harness/mod.rs b/nexus/db-queries/benches/harness/mod.rs index 6f7db092820..0b21cb743c0 100644 --- a/nexus/db-queries/benches/harness/mod.rs +++ b/nexus/db-queries/benches/harness/mod.rs @@ -35,15 +35,32 @@ struct ContentionQuery { const QUERIES: [ContentionQuery; 4] = [ ContentionQuery { - sql: "SELECT table_name, index_name, num_contention_events::TEXT FROM crdb_internal.cluster_contended_indexes", + sql: "SELECT + table_name, index_name, num_contention_events::TEXT + FROM crdb_internal.cluster_contended_indexes", description: "Indexes which are experiencing contention", }, ContentionQuery { - sql: "SELECT table_name,num_contention_events::TEXT FROM crdb_internal.cluster_contended_tables", + sql: "SELECT + table_name,num_contention_events::TEXT + FROM crdb_internal.cluster_contended_tables", description: "Tables which are experiencing contention", }, ContentionQuery { - sql: "WITH c AS (SELECT DISTINCT ON (table_id, index_id) table_id, index_id, num_contention_events AS events, cumulative_contention_time AS time FROM crdb_internal.cluster_contention_events) SELECT i.descriptor_name as table_name, i.index_name, c.events::TEXT, c.time::TEXT FROM crdb_internal.table_indexes AS i JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id ORDER BY c.time DESC LIMIT 10;", + sql: "WITH c AS + (SELECT DISTINCT ON (table_id, index_id) + table_id, + index_id, + num_contention_events AS events, + cumulative_contention_time AS time + FROM crdb_internal.cluster_contention_events) + SELECT + i.descriptor_name as table_name, + i.index_name, + c.events::TEXT, + c.time::TEXT FROM crdb_internal.table_indexes AS i + JOIN c ON i.descriptor_id = c.table_id AND i.index_id = c.index_id + ORDER BY c.time DESC LIMIT 10;", description: "Top ten longest contention events, grouped by table + index", }, ContentionQuery { diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index eb54156699a..47a647474d4 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -393,7 +393,20 @@ async fn bench_reservation( // For example: if the total number of vmms has no impact on the next provisioning // request, we should see similar durations for "100 vmms reserved" vs "1 vmm // reserved". However, if more vmms actually make reservation times slower, we'll see - // the "100 vmm" case take longer than the "1 vmm" case. The same goes for tasks: + // the "100 vmm" case take longer than the "1 vmm" case. + // + // The same is roughly true of tasks: as we add more tasks, unless the + // work is blocked on access to a resource (e.g., CPU, Disk, a single + // database table, etc), provisioning time should be constant. However, + // once the size of the incoming task queue grows larger than the work + // we can perform concurrently, we expect the provisioning time to + // increase linearly, and hopefully proportional to the cost of + // completing the VMM reservation with a single task. + // + // This is made explicit because it is not always true - incurring + // fallible retry logic, for example, can cause the cost of VMM + // reservation to increase SUPER-linearly with respect to the number of + // tasks, if additional tasks actively interfere with each other. total_duration += average_duration(all_tasks_duration, params.tasks); } // Destroy all our instance records (and their affinity group memberships) From e79fa5f0eb83aa6d7115e36c27f8d86f4761df3a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 24 Feb 2025 16:55:46 -0800 Subject: [PATCH 60/84] README --- nexus/db-queries/benches/README.adoc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 nexus/db-queries/benches/README.adoc diff --git a/nexus/db-queries/benches/README.adoc b/nexus/db-queries/benches/README.adoc new file mode 100644 index 00000000000..f4f1a716344 --- /dev/null +++ b/nexus/db-queries/benches/README.adoc @@ -0,0 +1,18 @@ +:showtitle: +:toc: left +:icons: font + += Benchmarks + +This directory contains benchmarks for database queries. + +These queries can be run with: + +[source,bash] +---- +cargo bench -p nexus-db-queries +---- + +Additionally, the "SHOW_CONTENTION" environment variable can be set to display +extra data from CockroachDB tables about contention statistics, if they +are available. From 01029f10591c8f12805f73588100baf84fb6567d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 11:36:50 -0800 Subject: [PATCH 61/84] Add issue --- nexus/db-queries/benches/sled_reservation.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/benches/sled_reservation.rs b/nexus/db-queries/benches/sled_reservation.rs index 47a647474d4..97eced60cab 100644 --- a/nexus/db-queries/benches/sled_reservation.rs +++ b/nexus/db-queries/benches/sled_reservation.rs @@ -446,7 +446,8 @@ fn sled_reservation_benchmark(c: &mut Criterion) { }, ], }, - // TODO create a test for "policy = Fail" groups. + // TODO(https://github.com/oxidecomputer/omicron/issues/7628): + // create a test for "policy = Fail" groups. ]; for grouping in &group_patterns { From b36be5776ed9cf52b29cb699afd809aa991b2366 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 11:49:04 -0800 Subject: [PATCH 62/84] new args to test helpers --- nexus/db-queries/src/db/datastore/sled.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 5231e9275de..a99e04d620d 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -2370,7 +2370,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2466,7 +2466,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2561,7 +2561,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; From 167a2d865af57072289ca279d721ea4a66cf2f2c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 12:06:04 -0800 Subject: [PATCH 63/84] Clippy --- nexus/db-queries/src/db/datastore/affinity.rs | 12 ++++++------ nexus/db-queries/src/db/datastore/sled.rs | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 3797560aca9..858778b89db 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -1711,7 +1711,7 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance(instance,), + external::AffinityGroupMember::Instance(instance), members[0].clone().into() ); @@ -1800,8 +1800,8 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance(instance,), - members[0].clone().into() + external::AntiAffinityGroupMember::Instance(instance), + members[0].clone() ); // We can delete the member and observe an empty member list @@ -1915,7 +1915,7 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance(instance,), + external::AffinityGroupMember::Instance(instance), members[0].clone().into() ); @@ -2029,8 +2029,8 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance(instance,), - members[0].clone().into() + external::AntiAffinityGroupMember::Instance(instance), + members[0].clone() ); // We can delete the member and observe an empty member list -- even diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 0ee75fbd478..29c5ab4b35e 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -2248,7 +2248,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2325,7 +2325,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 5; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2408,7 +2408,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 3; let sleds = create_sleds(&datastore, SLED_COUNT).await; @@ -2496,7 +2496,7 @@ pub(in crate::db::datastore) mod test { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); let (authz_project, _project) = - create_project(&opctx, &datastore).await; + create_project(&opctx, &datastore, "project").await; const SLED_COUNT: usize = 4; let sleds = create_sleds(&datastore, SLED_COUNT).await; From 17887d4db77a4fc7ab6e800aa4f187a3316c1962 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 12:44:50 -0800 Subject: [PATCH 64/84] please no panic --- nexus/db-queries/src/db/datastore/affinity.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 858778b89db..81f80d0ae7d 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -417,7 +417,7 @@ impl DataStore { i64::from(pagparams.limit.get()), ); - Ok(query + query .query::<(diesel::sql_types::Uuid, diesel::sql_types::Text)>() .load_async::<(Uuid, String)>( &*self.pool_connection_authorized(opctx).await?, @@ -428,16 +428,18 @@ impl DataStore { .map(|(id, label)| { use external::AntiAffinityGroupMember as Member; match label.as_str() { - "affinity_group" => Member::AffinityGroup( + "affinity_group" => Ok(Member::AffinityGroup( AffinityGroupUuid::from_untyped_uuid(id), - ), - "instance" => { - Member::Instance(InstanceUuid::from_untyped_uuid(id)) - } - other => panic!("Unexpected label from query: {other}"), + )), + "instance" => Ok(Member::Instance( + InstanceUuid::from_untyped_uuid(id), + )), + other => Err(external::Error::internal_error(&format!( + "Unexpected label from database query: {other}" + ))), } }) - .collect()) + .collect() } pub async fn affinity_group_member_view( From 5865732ffb42123e660188e79c4bc625eeadf6b6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 13:06:36 -0800 Subject: [PATCH 65/84] Safer group-to-group rule respecting --- nexus/db-queries/src/db/datastore/affinity.rs | 46 ++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 81f80d0ae7d..2562429728b 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -821,7 +821,9 @@ impl DataStore { let err = err.clone(); use db::schema::anti_affinity_group::dsl as anti_affinity_group_dsl; use db::schema::affinity_group::dsl as affinity_group_dsl; - use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + use db::schema::affinity_group_instance_membership::dsl as a_instance_membership_dsl; + use db::schema::anti_affinity_group_affinity_membership::dsl as aa_affinity_membership_dsl; + use db::schema::sled_resource_vmm::dsl as resource_dsl; async move { // Check that the anti-affinity group exists @@ -861,12 +863,44 @@ impl DataStore { }) })?; - // TODO: It's possible that the affinity group has members - // which are already running. We should probably check this, - // and prevent it, otherwise we could circumvent "policy = - // fail" stances. - diesel::insert_into(membership_dsl::anti_affinity_group_affinity_membership) + // Check that the affinity group's members are not reserved. + let has_reservation: bool = diesel::select( + diesel::dsl::exists( + a_instance_membership_dsl::affinity_group_instance_membership + .inner_join( + resource_dsl::sled_resource_vmm + .on(resource_dsl::instance_id.eq(a_instance_membership_dsl::instance_id.nullable())) + ) + .filter(a_instance_membership_dsl::group_id.eq( + affinity_group_id.into_untyped_uuid() + )) + ) + ).get_result_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + }) + })?; + + // NOTE: This check prevents us from violating affinity rules with "policy = + // fail" stances, but it is possible that running instances already would + // satisfy the affinity rules proposed by this new group addition. + // + // It would be possible to remove this error if we replaced it with affinity + // rule checks. + if has_reservation { + return Err(err.bail(Error::invalid_request( + "Affinity group with running instances cannot be \ + added to anti-affinity group. Try stopping them first.".to_string() + ))); + } + + diesel::insert_into(aa_affinity_membership_dsl::anti_affinity_group_affinity_membership) .values(AntiAffinityGroupAffinityMembership::new( AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), affinity_group_id, From 6ab1c67c4129d56945de4df4c86af5814607d56a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 15:02:19 -0800 Subject: [PATCH 66/84] Add integration test for group of group --- nexus/tests/integration_tests/affinity.rs | 313 ++++++++++++++-------- 1 file changed, 206 insertions(+), 107 deletions(-) diff --git a/nexus/tests/integration_tests/affinity.rs b/nexus/tests/integration_tests/affinity.rs index 1549db51cba..89a5aa80a9f 100644 --- a/nexus/tests/integration_tests/affinity.rs +++ b/nexus/tests/integration_tests/affinity.rs @@ -162,78 +162,60 @@ impl ProjectScopedApiHelper<'_, T> { object_get_error(&self.client, &url, status).await } - async fn group_member_add(&self, group: &str, instance: &str) -> T::Member { - let url = group_member_instance_url( - T::URL_COMPONENT, - self.project, - group, - instance, - ); + async fn group_member_add( + &self, + group: &str, + member: &str, + ) -> T::Member { + let url = M::url(T::URL_COMPONENT, self.project, group, member); object_create(&self.client, &url, &()).await } - async fn group_member_add_expect_error( + async fn group_member_add_expect_error( &self, group: &str, - instance: &str, + member: &str, status: StatusCode, ) -> HttpErrorResponseBody { - let url = group_member_instance_url( - T::URL_COMPONENT, - self.project, - group, - instance, - ); + let url = M::url(T::URL_COMPONENT, self.project, group, member); object_create_error(&self.client, &url, &(), status).await } - async fn group_member_get(&self, group: &str, instance: &str) -> T::Member { - let url = group_member_instance_url( - T::URL_COMPONENT, - self.project, - group, - instance, - ); + async fn group_member_get( + &self, + group: &str, + member: &str, + ) -> T::Member { + let url = M::url(T::URL_COMPONENT, self.project, group, member); object_get(&self.client, &url).await } - async fn group_member_get_expect_error( + async fn group_member_get_expect_error( &self, group: &str, - instance: &str, + member: &str, status: StatusCode, ) -> HttpErrorResponseBody { - let url = group_member_instance_url( - T::URL_COMPONENT, - self.project, - group, - instance, - ); + let url = M::url(T::URL_COMPONENT, self.project, group, member); object_get_error(&self.client, &url, status).await } - async fn group_member_delete(&self, group: &str, instance: &str) { - let url = group_member_instance_url( - T::URL_COMPONENT, - self.project, - group, - instance, - ); + async fn group_member_delete( + &self, + group: &str, + member: &str, + ) { + let url = M::url(T::URL_COMPONENT, self.project, group, member); object_delete(&self.client, &url).await } - async fn group_member_delete_expect_error( + async fn group_member_delete_expect_error( &self, group: &str, - instance: &str, + member: &str, status: StatusCode, ) -> HttpErrorResponseBody { - let url = group_member_instance_url( - T::URL_COMPONENT, - self.project, - group, - instance, - ); + let url = M::url(T::URL_COMPONENT, self.project, group, member); object_delete_error(&self.client, &url, status).await } } @@ -416,14 +398,44 @@ fn group_members_url(ty: &str, project: Option<&str>, group: &str) -> String { format!("/v1/{ty}/{group}/members{query_params}") } -fn group_member_instance_url( - ty: &str, - project: Option<&str>, - group: &str, - instance: &str, -) -> String { - let query_params = project_query_param_suffix(project); - format!("/v1/{ty}/{group}/members/instance/{instance}{query_params}") +/// Describes shared logic between "things that can be group members". +trait GroupMemberish { + fn url( + ty: &str, + project: Option<&str>, + group: &str, + member: &str, + ) -> String; +} + +struct MemberInstance {} + +impl GroupMemberish for MemberInstance { + fn url( + ty: &str, + project: Option<&str>, + group: &str, + member: &str, + ) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}/members/instance/{member}{query_params}") + } +} + +struct MemberAffinityGroup {} + +impl GroupMemberish for MemberAffinityGroup { + fn url( + ty: &str, + project: Option<&str>, + group: &str, + member: &str, + ) -> String { + let query_params = project_query_param_suffix(project); + format!( + "/v1/{ty}/{group}/members/affinity-group/{member}{query_params}" + ) + } } #[nexus_test(extra_sled_agents = 2)] @@ -484,7 +496,10 @@ async fn test_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { // Add these instances to an affinity group for instance in &instances { project_api - .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) + .group_member_add::( + GROUP_NAME, + &instance.identity.name.to_string(), + ) .await; } @@ -495,7 +510,10 @@ async fn test_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { // We can also list each member for instance in &instances { project_api - .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .group_member_get::( + GROUP_NAME, + instance.identity.name.as_str(), + ) .await; } @@ -543,8 +561,8 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { const PROJECT_NAME: &'static str = "test-project"; const GROUP_NAME: &'static str = "group"; + const AFFINITY_GROUP_NAME: &'static str = "a-group"; const EXPECTED_SLEDS: usize = 3; - const INSTANCE_COUNT: usize = EXPECTED_SLEDS; let api = ApiHelper::new(external_client); @@ -562,69 +580,118 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { create_default_ip_pool(&external_client).await; api.create_project(PROJECT_NAME).await; - let project_api = api.use_project::(PROJECT_NAME); + let aa_project_api = api.use_project::(PROJECT_NAME); + let a_project_api = api.use_project::(PROJECT_NAME); - let mut instances = Vec::new(); - for i in 0..INSTANCE_COUNT { - instances.push( - project_api - .create_stopped_instance(&format!("test-instance-{i}")) + // Create both stopped instances and some affinity groups. + // + // All but two of the instances are going to be anti-affine from each other. + let mut aa_instances = Vec::new(); + for i in 0..EXPECTED_SLEDS - 1 { + aa_instances.push( + aa_project_api + .create_stopped_instance(&format!("test-aa-instance-{i}")) .await, ); } + // These two instances will be affine with each other, but anti-affine from + // everything else (indirectly) through their affinity group. + let mut a_instances = Vec::new(); + a_project_api.group_create(AFFINITY_GROUP_NAME).await; + for i in 0..2 { + let instance = a_project_api + .create_stopped_instance(&format!("test-a-instance-{i}")) + .await; + a_project_api + .group_member_add::( + AFFINITY_GROUP_NAME, + &instance.identity.name.to_string(), + ) + .await; + a_instances.push(instance); + } // When we start, we observe no anti-affinity groups - let groups = project_api.groups_list().await; + let groups = aa_project_api.groups_list().await; assert!(groups.is_empty()); // We can now create a group and observe it - let group = project_api.group_create(GROUP_NAME).await; + let group = aa_project_api.group_create(GROUP_NAME).await; // We can list it and also GET the group specifically - let groups = project_api.groups_list().await; + let groups = aa_project_api.groups_list().await; assert_eq!(groups.len(), 1); assert_eq!(groups[0].identity.id, group.identity.id); - let observed_group = project_api.group_get(GROUP_NAME).await; + let observed_group = aa_project_api.group_get(GROUP_NAME).await; assert_eq!(observed_group.identity.id, group.identity.id); // List all members of the anti-affinity group (expect nothing) - let members = project_api.group_members_list(GROUP_NAME).await; + let members = aa_project_api.group_members_list(GROUP_NAME).await; assert!(members.is_empty()); // Add these instances to the anti-affinity group - for instance in &instances { - project_api - .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) + for instance in &aa_instances { + aa_project_api + .group_member_add::( + GROUP_NAME, + &instance.identity.name.to_string(), + ) .await; } + // Add the affinity group to the anti-affinity group + aa_project_api + .group_member_add::( + GROUP_NAME, + AFFINITY_GROUP_NAME, + ) + .await; - // List members again (expect all instances) - let members = project_api.group_members_list(GROUP_NAME).await; - assert_eq!(members.len(), instances.len()); + // List members again (expect all instances, and the affinity group) + let members = aa_project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), aa_instances.len() + 1); // We can also list each member - for instance in &instances { - project_api - .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + for instance in &aa_instances { + aa_project_api + .group_member_get::( + GROUP_NAME, + instance.identity.name.as_str(), + ) .await; } + aa_project_api + .group_member_get::( + GROUP_NAME, + AFFINITY_GROUP_NAME, + ) + .await; // Start the instances we created earlier. // // We don't actually care that they're "running" from the perspective of the // simulated sled agent, we just want placement to be triggered from Nexus. - for instance in &instances { + for instance in &aa_instances { + api.start_instance(&instance).await; + } + for instance in &a_instances { api.start_instance(&instance).await; } - let mut expected_instances = instances + let mut expected_aa_instances = aa_instances + .iter() + .map(|instance| instance.identity.id) + .collect::>(); + let mut expected_a_instances = a_instances .iter() .map(|instance| instance.identity.id) .collect::>(); - // We expect that each sled will have a since instance, as all of the + // We expect that each sled will have a single instance, as all of the // instances will want to be anti-located from each other. + // + // The only exception will be a single sled containing both of the + // affine instances, which should be separate from all others. for sled in &sleds { let observed_instances = api .sled_instance_list(&sled.identity.id.to_string()) @@ -633,22 +700,39 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { .map(|sled_instance| sled_instance.identity.id) .collect::>(); - assert_eq!( - observed_instances.len(), - 1, - "All instances should be placed on distinct sleds" - ); - - assert!( - expected_instances.remove(&observed_instances[0]), - "The instance {} was observed on multiple sleds", - observed_instances[0] - ); + match observed_instances.len() { + 1 => { + // This should be one of the anti-affine instances + assert!( + expected_aa_instances.remove(&observed_instances[0]), + "The instance {} was observed too many times", + observed_instances[0] + ); + } + 2 => { + // This should be one of the affine instances, anti-affine from + // others because of their affinity groups + for observed_instance in observed_instances { + assert!( + expected_a_instances.remove(&observed_instance), + "The instance {} was observed too many times", + observed_instance + ); + } + } + _ => panic!( + "Unexpected instance count on sled: {observed_instances:?}" + ), + } } assert!( - expected_instances.is_empty(), - "Did not find allocations for some instances: {expected_instances:?}" + expected_aa_instances.is_empty(), + "Did not find allocations for some anti-affine instances: {expected_aa_instances:?}" + ); + assert!( + expected_a_instances.is_empty(), + "Did not find allocations for some affine instances: {expected_a_instances:?}" ); } @@ -702,9 +786,11 @@ async fn test_group_crud(client: &ClientTestContext) { // Add the instance to the affinity group let instance_name = &instance.identity.name.to_string(); - project_api.group_member_add(GROUP_NAME, &instance_name).await; + project_api + .group_member_add::(GROUP_NAME, &instance_name) + .await; let response = project_api - .group_member_add_expect_error( + .group_member_add_expect_error::( GROUP_NAME, &instance_name, StatusCode::BAD_REQUEST, @@ -723,13 +809,18 @@ async fn test_group_crud(client: &ClientTestContext) { let members = project_api.group_members_list(GROUP_NAME).await; assert_eq!(members.len(), 1); project_api - .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .group_member_get::( + GROUP_NAME, + instance.identity.name.as_str(), + ) .await; // Delete the member, observe that it is gone - project_api.group_member_delete(GROUP_NAME, &instance_name).await; project_api - .group_member_delete_expect_error( + .group_member_delete::(GROUP_NAME, &instance_name) + .await; + project_api + .group_member_delete_expect_error::( GROUP_NAME, &instance_name, StatusCode::NOT_FOUND, @@ -738,7 +829,7 @@ async fn test_group_crud(client: &ClientTestContext) { let members = project_api.group_members_list(GROUP_NAME).await; assert_eq!(members.len(), 0); project_api - .group_member_get_expect_error( + .group_member_get_expect_error::( GROUP_NAME, &instance_name, StatusCode::NOT_FOUND, @@ -840,21 +931,29 @@ async fn test_group_project_selector( // Group Members can be added by name or UUID let instance_name = instance.identity.name.as_str(); let instance_id = instance.identity.id.to_string(); - project_api.group_member_add(GROUP_NAME, instance_name).await; - project_api.group_member_delete(GROUP_NAME, instance_name).await; - no_project_api.group_member_add(&group_id, &instance_id).await; - no_project_api.group_member_delete(&group_id, &instance_id).await; + project_api + .group_member_add::(GROUP_NAME, instance_name) + .await; + project_api + .group_member_delete::(GROUP_NAME, instance_name) + .await; + no_project_api + .group_member_add::(&group_id, &instance_id) + .await; + no_project_api + .group_member_delete::(&group_id, &instance_id) + .await; // Trying to use UUIDs with the project selector is invalid project_api - .group_member_add_expect_error( + .group_member_add_expect_error::( GROUP_NAME, &instance_id, StatusCode::BAD_REQUEST, ) .await; project_api - .group_member_add_expect_error( + .group_member_add_expect_error::( &group_id, instance_name, StatusCode::BAD_REQUEST, @@ -863,21 +962,21 @@ async fn test_group_project_selector( // Using any names without the project selector is invalid no_project_api - .group_member_add_expect_error( + .group_member_add_expect_error::( GROUP_NAME, &instance_id, StatusCode::BAD_REQUEST, ) .await; no_project_api - .group_member_add_expect_error( + .group_member_add_expect_error::( &group_id, instance_name, StatusCode::BAD_REQUEST, ) .await; no_project_api - .group_member_add_expect_error( + .group_member_add_expect_error::( GROUP_NAME, instance_name, StatusCode::BAD_REQUEST, From 0cb05b5271e80f19e62799544c75c42a4ac21aac Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 25 Feb 2025 16:03:50 -0800 Subject: [PATCH 67/84] Expanding testing, improving deletion semantics --- nexus/db-queries/src/db/datastore/affinity.rs | 419 +++++++++++++++--- 1 file changed, 367 insertions(+), 52 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 2562429728b..842062bc330 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -228,17 +228,35 @@ impl DataStore { })?; // Ensure all memberships in the affinity group are deleted - use db::schema::affinity_group_instance_membership::dsl as member_dsl; - diesel::delete(member_dsl::affinity_group_instance_membership) - .filter(member_dsl::group_id.eq(authz_affinity_group.id())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; + { + use db::schema::affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + } + // If this affinity group is a member in other anti-affinity + // groups, remove those memberships + // + // TODO: This needs testing + { + use db::schema::anti_affinity_group_affinity_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_affinity_membership) + .filter(member_dsl::affinity_group_id.eq(authz_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + } Ok(()) } }) @@ -315,17 +333,31 @@ impl DataStore { }) })?; - // Ensure all memberships in the anti affinity group are deleted - use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; - diesel::delete(member_dsl::anti_affinity_group_instance_membership) - .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; + // Ensure all memberships in the anti-affinity group are deleted + { + use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + } + { + use db::schema::anti_affinity_group_affinity_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_affinity_membership) + .filter(member_dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + } Ok(()) } @@ -896,7 +928,7 @@ impl DataStore { if has_reservation { return Err(err.bail(Error::invalid_request( "Affinity group with running instances cannot be \ - added to anti-affinity group. Try stopping them first.".to_string() + added to an anti-affinity group. Try stopping them first.".to_string() ))); } @@ -2090,6 +2122,167 @@ mod tests { logctx.cleanup_successful(); } + #[tokio::test] + async fn anti_affinity_group_membership_add_remove_group_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_remove_group_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Also, create an affinity group. It'll be a member within the + // anti-affinity group. + let affinity_group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-affinity-group", + ) + .await + .unwrap(); + let affinity_group_id = + AffinityGroupUuid::from_untyped_uuid(affinity_group.id()); + + let (.., authz_aa_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + let (.., authz_a_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(affinity_group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_stopped_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the affinity group + datastore + .affinity_group_member_add( + &opctx, + &authz_a_group, + external::AffinityGroupMember::Instance(instance), + ) + .await + .unwrap(); + + // Reserve the VMM for the instance. + allocate_instance_reservation(&datastore, instance).await; + + // Now, we cannot add the affinity group to the anti-affinity group + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_aa_group, + external::AntiAffinityGroupMember::AffinityGroup(affinity_group_id), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to anti-affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Affinity group with running instances cannot be added to an anti-affinity group" + ), + "{err:?}" + ); + + // If we have no reservation for the affinity group, we can add it to the + // anti-affinity group. + delete_instance_reservation(&datastore, instance).await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_aa_group, + external::AntiAffinityGroupMember::AffinityGroup( + affinity_group_id, + ), + ) + .await + .unwrap(); + + // Now we can reserve a sled for the instance once more. + allocate_instance_reservation(&datastore, instance).await; + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::AffinityGroup(affinity_group_id), + members[0].clone() + ); + + // We can delete the member and observe an empty member list -- even + // though it has an instance which is running! + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_aa_group, + external::AntiAffinityGroupMember::AffinityGroup( + affinity_group_id, + ), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn affinity_group_delete_group_deletes_members() { // Setup @@ -2182,7 +2375,16 @@ mod tests { .await .unwrap(); - let (.., authz_group) = LookupPath::new(opctx, datastore) + let affinity_group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-affinity-group", + ) + .await + .unwrap(); + + let (.., authz_aa_group) = LookupPath::new(opctx, datastore) .anti_affinity_group_id(group.id()) .lookup_for(authz::Action::Modify) .await @@ -2195,7 +2397,11 @@ mod tests { direction: dropshot::PaginationOrder::Ascending, }; let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) .await .unwrap(); assert!(members.is_empty()); @@ -2211,24 +2417,42 @@ mod tests { datastore .anti_affinity_group_member_add( &opctx, - &authz_group, + &authz_aa_group, external::AntiAffinityGroupMember::Instance(instance), ) .await .unwrap(); + // Also, add the affinity member to the anti-affinity group as a member + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_aa_group, + external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), + ), + ) + .await + .unwrap(); // Delete the group datastore - .anti_affinity_group_delete(&opctx, &authz_group) + .anti_affinity_group_delete(&opctx, &authz_aa_group) .await .unwrap(); - // Confirm that no instance members exist + // Confirm that no group members exist let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) .await .unwrap(); - assert!(members.is_empty()); + assert!( + members.is_empty(), + "No members should exist, but these do: {members:?}" + ); // Clean up. db.terminate().await; @@ -2563,17 +2787,60 @@ mod tests { let (authz_project, ..) = create_project(&opctx, &datastore, "my-project").await; + enum Member { + Instance, + AffinityGroup, + } + + impl Member { + fn resource_type(&self) -> ResourceType { + match self { + Member::Instance => ResourceType::Instance, + Member::AffinityGroup => ResourceType::AffinityGroup, + } + } + } + struct TestArgs { // Does the group exist? group: bool, - // Does the instance exist? - instance: bool, + // What's the type of the member? + member_type: Member, + // Does the member exist? + member: bool, } let args = [ - TestArgs { group: false, instance: false }, - TestArgs { group: true, instance: false }, - TestArgs { group: false, instance: true }, + TestArgs { + group: false, + member_type: Member::Instance, + member: false, + }, + TestArgs { + group: true, + member_type: Member::Instance, + member: false, + }, + TestArgs { + group: false, + member_type: Member::Instance, + member: true, + }, + TestArgs { + group: false, + member_type: Member::AffinityGroup, + member: false, + }, + TestArgs { + group: true, + member_type: Member::AffinityGroup, + member: false, + }, + TestArgs { + group: false, + member_type: Member::AffinityGroup, + member: true, + }, ]; for arg in args { @@ -2600,7 +2867,7 @@ mod tests { .unwrap(); } - // Create an instance, and maybe delete it. + // Create an instance let instance = create_stopped_instance_record( &opctx, &datastore, @@ -2608,32 +2875,78 @@ mod tests { "my-instance", ) .await; + let mut instance_exists = true; + + // Create an affinity group + let affinity_group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-affinity-group", + ) + .await + .unwrap(); + let mut affinity_group_exists = true; + let (.., authz_instance) = LookupPath::new(opctx, datastore) .instance_id(instance.into_untyped_uuid()) .lookup_for(authz::Action::Modify) .await .unwrap(); - if !arg.instance { - datastore - .project_delete_instance(&opctx, &authz_instance) - .await - .unwrap(); + let (.., authz_affinity_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(affinity_group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.member { + match arg.member_type { + Member::Instance => { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + instance_exists = false; + } + Member::AffinityGroup => { + datastore + .affinity_group_delete( + &opctx, + &authz_affinity_group, + ) + .await + .unwrap(); + affinity_group_exists = false; + } + } } - // Try to add the instance to the group. + // Try to add the member to the group. // // Expect to see specific errors, depending on whether or not the - // group/instance exist. + // group/member exist. + let member = match arg.member_type { + Member::Instance => { + external::AntiAffinityGroupMember::Instance(instance) + } + Member::AffinityGroup => { + external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid( + affinity_group.id(), + ), + ) + } + }; let err = datastore .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + member.clone(), ) .await .expect_err("Should have failed"); - match (arg.group, arg.instance) { + match (arg.group, arg.member) { (false, _) => { assert!( matches!(err, Error::ObjectNotFound { @@ -2646,7 +2959,7 @@ mod tests { assert!( matches!(err, Error::ObjectNotFound { type_name, .. - } if type_name == ResourceType::Instance), + } if type_name == arg.member_type.resource_type()), "{err:?}" ); } @@ -2657,14 +2970,10 @@ mod tests { // Do the same thing, but for group membership removal. let err = datastore - .anti_affinity_group_member_delete( - &opctx, - &authz_group, - external::AntiAffinityGroupMember::Instance(instance), - ) + .anti_affinity_group_member_delete(&opctx, &authz_group, member) .await .expect_err("Should have failed"); - match (arg.group, arg.instance) { + match (arg.group, arg.member) { (false, _) => { assert!( matches!(err, Error::ObjectNotFound { @@ -2687,12 +2996,18 @@ mod tests { } // Cleanup, if we actually created anything. - if arg.instance { + if instance_exists { datastore .project_delete_instance(&opctx, &authz_instance) .await .unwrap(); } + if affinity_group_exists { + datastore + .affinity_group_delete(&opctx, &authz_affinity_group) + .await + .unwrap(); + } if arg.group { datastore .anti_affinity_group_delete(&opctx, &authz_group) From 9ad231328cf82e6067456c5091af79d0b68156dd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 26 Feb 2025 12:14:12 -0800 Subject: [PATCH 68/84] More testing --- nexus/db-queries/src/db/datastore/affinity.rs | 97 ++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 842062bc330..76385b7b5a7 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -243,8 +243,6 @@ impl DataStore { // If this affinity group is a member in other anti-affinity // groups, remove those memberships - // - // TODO: This needs testing { use db::schema::anti_affinity_group_affinity_membership::dsl as member_dsl; diesel::delete(member_dsl::anti_affinity_group_affinity_membership) @@ -2459,6 +2457,101 @@ mod tests { logctx.cleanup_successful(); } + // Since this name is gnarly, just to be clear: + // - Affinity groups can be "members" within anti-affinity groups + // - If one of these memberships is alive when the affinity group is + // deleted, that membership should be automatically removed + // + // Basically, do not keep around a reference to a dead affinity group. + #[tokio::test] + async fn affinity_group_delete_group_deletes_membership_in_anti_affinity_groups( + ) { + // Setup + let logctx = + dev::test_setup_log("affinity_group_delete_group_deletes_membership_in_anti_affinity_groups"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and the groups + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let affinity_group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "affinity", + ) + .await + .unwrap(); + let anti_affinity_group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "anti-affinity", + ) + .await + .unwrap(); + + let (.., authz_a_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(affinity_group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + let (.., authz_aa_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(anti_affinity_group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Add the affinity group to the anti-affinity group. + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), + ); + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_aa_group, + member.clone(), + ) + .await + .unwrap(); + + // Right now, the affinity group is observable + datastore + .anti_affinity_group_member_view( + &opctx, + &authz_aa_group, + member.clone(), + ) + .await + .expect("Group member should be visible - we just added it"); + + // Delete the affinity group (the member) + datastore.affinity_group_delete(&opctx, &authz_a_group).await.unwrap(); + + // The affinity group membership should have been revoked + let err = datastore + .anti_affinity_group_member_view( + &opctx, + &authz_aa_group, + member.clone(), + ) + .await + .expect_err("Group member should no longer exist"); + assert!( + matches!( + err, + Error::ObjectNotFound { type_name, .. } + if type_name == ResourceType::AntiAffinityGroupMember + ), + "Unexpected error: {err:?}" + ); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn affinity_group_delete_instance_deletes_membership() { // Setup From 56f300a7129734f603e1052ae2240b0237edebb1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 26 Feb 2025 13:12:10 -0800 Subject: [PATCH 69/84] Fix listing, add listing test --- common/src/api/external/mod.rs | 4 +- nexus/db-queries/src/db/datastore/affinity.rs | 193 +++++++++++++++++- 2 files changed, 185 insertions(+), 12 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index a5af4647b7e..fa30f94bb20 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1337,7 +1337,9 @@ impl SimpleIdentity for AffinityGroupMember { /// /// Membership in a group is not exclusive - members may belong to multiple /// affinity / anti-affinity groups. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Hash, Eq, +)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { /// An affinity group belonging to this group, identified by UUID. diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 76385b7b5a7..550d6a3970d 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -406,9 +406,14 @@ impl DataStore { ) -> ListResultVec { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + let asc = match pagparams.direction { + dropshot::PaginationOrder::Ascending => true, + dropshot::PaginationOrder::Descending => false, + }; + let mut query = QueryBuilder::new() .sql( - " + "SELECT id,label FROM ( SELECT instance_id as id, 'instance' as label FROM anti_affinity_group_instance_membership WHERE group_id = ", @@ -424,24 +429,23 @@ impl DataStore { ) .param() .bind::(authz_anti_affinity_group.id()) - .sql(" "); - - let (sort, cmp) = match pagparams.direction { - dropshot::PaginationOrder::Ascending => (" ORDER BY id ASC ", ">"), - dropshot::PaginationOrder::Descending => { - (" ORDER BY id DESC ", "<") - } - }; + .sql(") "); if let Some(id) = pagparams.marker { query = query .sql("WHERE id ") - .sql(cmp) + .sql(if asc { ">" } else { "<" }) .sql(" ") .param() .bind::(*id); }; - query = query.sql(sort); + query = query.sql(" ORDER BY id "); + if asc { + query = query.sql("ASC "); + } else { + query = query.sql("DESC "); + } + query = query.sql(" LIMIT ").param().bind::( i64::from(pagparams.limit.get()), @@ -1230,6 +1234,7 @@ mod tests { use nexus_types::external_api::params; use omicron_common::api::external::{ self, ByteCount, DataPageParams, IdentityMetadataCreateParams, + SimpleIdentity, }; use omicron_test_utils::dev; use omicron_uuid_kinds::GenericUuid; @@ -1890,6 +1895,172 @@ mod tests { logctx.cleanup_successful(); } + // Anti-affinity group member listing has a slightly more complicated + // implementation, because it queries multiple tables and UNIONs them + // together. + // + // This test exists to validate that manual implementation. + #[tokio::test] + async fn anti_affinity_group_membership_list_extended() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_group_membership_list_extended"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_aa_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert!(members.is_empty()); + + // Add some groups and instances, so we have data to list over. + + const INSTANCE_COUNT: usize = 3; + const AFFINITY_GROUP_COUNT: usize = 3; + + let mut members = Vec::new(); + + for i in 0..INSTANCE_COUNT { + let instance = create_stopped_instance_record( + &opctx, + &datastore, + &authz_project, + &format!("instance-{i}"), + ) + .await; + + // Add the instance as a member to the group + let member = external::AntiAffinityGroupMember::Instance(instance); + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_aa_group, + member.clone(), + ) + .await + .unwrap(); + members.push(member); + } + + for i in 0..AFFINITY_GROUP_COUNT { + let affinity_group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + &format!("affinity-{i}"), + ) + .await + .unwrap(); + + // Add the instance as a member to the group + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), + ); + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_aa_group, + member.clone(), + ) + .await + .unwrap(); + members.push(member); + } + + // Order by UUID, regardless of member type + members.sort_unstable_by(|m1, m2| m1.id().cmp(&m2.id())); + + // We can list all members + let mut pagparams = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members); + + // We can paginate over the results + let marker = members[2].id(); + pagparams.marker = Some(&marker); + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members[3..]); + + // We can list limited results + pagparams.marker = Some(&marker); + pagparams.limit = NonZeroU32::new(2).unwrap(); + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members[3..5]); + + // We can list in descending order too + members.reverse(); + pagparams.marker = None; + pagparams.limit = NonZeroU32::new(100).unwrap(); + pagparams.direction = dropshot::PaginationOrder::Descending; + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn affinity_group_membership_add_remove_instance_with_vmm() { // Setup From 05ebab788d9fb6a861889405c3e568b99b8bdcbd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 26 Feb 2025 13:27:31 -0800 Subject: [PATCH 70/84] clipperoni --- nexus/db-queries/src/db/datastore/affinity.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 550d6a3970d..b8ea09161ab 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -1997,7 +1997,7 @@ mod tests { } // Order by UUID, regardless of member type - members.sort_unstable_by(|m1, m2| m1.id().cmp(&m2.id())); + members.sort_unstable_by_key(|m1| m1.id()); // We can list all members let mut pagparams = DataPageParams { From ec9a490afdd2b6754eaee2aee3eb18d9ef766a0c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 26 Feb 2025 14:15:23 -0800 Subject: [PATCH 71/84] AffinityGroupMember exposes member name --- common/src/api/external/mod.rs | 7 +- nexus/db-model/src/affinity.rs | 12 +- nexus/db-queries/src/db/datastore/affinity.rs | 145 ++++++-------- .../src/db/pub_test_utils/helpers.rs | 8 +- nexus/external-api/output/nexus_tags.txt | 3 + nexus/src/app/affinity.rs | 38 ++-- openapi/nexus.json | 180 +++++++++++++++++- 7 files changed, 270 insertions(+), 123 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fa30f94bb20..229aa4776ac 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1321,14 +1321,15 @@ pub enum FailureDomain { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AffinityGroupMember { - /// An instance belonging to this group, identified by UUID. - Instance(InstanceUuid), + /// An instance belonging to this group + Instance { id: InstanceUuid, name: Name }, } +// TODO: Revisit this impl.. it was originally created for UUID-only members impl SimpleIdentity for AffinityGroupMember { fn id(&self) -> Uuid { match self { - AffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), + AffinityGroupMember::Instance { id, .. } => *id.as_untyped_uuid(), } } } diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index 89214f85778..4999a12dc4e 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -227,11 +227,15 @@ impl AffinityGroupInstanceMembership { pub fn new(group_id: AffinityGroupUuid, instance_id: InstanceUuid) -> Self { Self { group_id: group_id.into(), instance_id: instance_id.into() } } -} -impl From for external::AffinityGroupMember { - fn from(member: AffinityGroupInstanceMembership) -> Self { - Self::Instance(member.instance_id.into()) + pub fn to_external( + self, + member_name: external::Name, + ) -> external::AffinityGroupMember { + external::AffinityGroupMember::Instance { + id: self.instance_id.into(), + name: member_name, + } } } diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index b8ea09161ab..3e562a77bed 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -375,10 +375,11 @@ impl DataStore { opctx: &OpContext, authz_affinity_group: &authz::AffinityGroup, pagparams: &PaginatedBy<'_>, - ) -> ListResultVec { + ) -> ListResultVec<(AffinityGroupInstanceMembership, Name)> { opctx.authorize(authz::Action::Read, authz_affinity_group).await?; use db::schema::affinity_group_instance_membership::dsl; + use db::schema::instance::dsl as instance_dsl; match pagparams { PaginatedBy::Id(pagparams) => paginated( dsl::affinity_group_instance_membership, @@ -392,7 +393,14 @@ impl DataStore { } } .filter(dsl::group_id.eq(authz_affinity_group.id())) - .select(AffinityGroupInstanceMembership::as_select()) + .inner_join( + instance_dsl::instance.on(instance_dsl::id.eq(dsl::instance_id)), + ) + .filter(instance_dsl::time_deleted.is_null()) + .select(( + AffinityGroupInstanceMembership::as_select(), + instance_dsl::name, + )) .load_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) @@ -476,27 +484,32 @@ impl DataStore { .collect() } - pub async fn affinity_group_member_view( + pub async fn affinity_group_member_instance_view( &self, opctx: &OpContext, authz_affinity_group: &authz::AffinityGroup, - member: external::AffinityGroupMember, + instance_id: InstanceUuid, ) -> Result { opctx.authorize(authz::Action::Read, authz_affinity_group).await?; let conn = self.pool_connection_authorized(opctx).await?; - let instance_id = match member { - external::AffinityGroupMember::Instance(id) => id, - }; - use db::schema::affinity_group_instance_membership::dsl; + use db::schema::instance::dsl as instance_dsl; dsl::affinity_group_instance_membership .filter(dsl::group_id.eq(authz_affinity_group.id())) .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) - .select(AffinityGroupInstanceMembership::as_select()) - .get_result_async(&*conn) + .inner_join( + instance_dsl::instance + .on(instance_dsl::id.eq(dsl::instance_id)), + ) + .filter(instance_dsl::time_deleted.is_null()) + .select(( + AffinityGroupInstanceMembership::as_select(), + instance_dsl::name, + )) + .get_result_async::<(AffinityGroupInstanceMembership, Name)>(&*conn) .await - .map(|m| m.into()) + .map(|(member, name)| member.to_external(name.into())) .map_err(|e| { public_error_from_diesel( e, @@ -573,21 +586,17 @@ impl DataStore { } } - pub async fn affinity_group_member_add( + pub async fn affinity_group_member_instance_add( &self, opctx: &OpContext, authz_affinity_group: &authz::AffinityGroup, - member: external::AffinityGroupMember, + instance_id: InstanceUuid, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; - let instance_id = match member { - external::AffinityGroupMember::Instance(id) => id, - }; - let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("affinity_group_member_add") + self.transaction_retry_wrapper("affinity_group_member_instance_add") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::affinity_group::dsl as group_dsl; @@ -995,21 +1004,17 @@ impl DataStore { Ok(()) } - pub async fn affinity_group_member_delete( + pub async fn affinity_group_member_instance_delete( &self, opctx: &OpContext, authz_affinity_group: &authz::AffinityGroup, - member: external::AffinityGroupMember, + instance_id: InstanceUuid, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; - let instance_id = match member { - external::AffinityGroupMember::Instance(id) => id, - }; - let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("affinity_group_member_delete") + self.transaction_retry_wrapper("affinity_group_member_instance_delete") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::affinity_group::dsl as group_dsl; @@ -1767,11 +1772,7 @@ mod tests { // Add the instance as a member to the group datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .unwrap(); @@ -1779,19 +1780,19 @@ mod tests { let members = datastore .affinity_group_member_list(&opctx, &authz_group, &pagbyid) .await - .unwrap(); + .unwrap() + .into_iter() + .map(|(m, _)| m.instance_id.into()) + .collect::>(); assert_eq!(members.len(), 1); - assert_eq!( - external::AffinityGroupMember::Instance(instance), - members[0].clone().into() - ); + assert_eq!(instance, members[0]); // We can delete the member and observe an empty member list datastore - .affinity_group_member_delete( + .affinity_group_member_instance_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -2114,11 +2115,7 @@ mod tests { // Cannot add the instance to the group while it's running. let err = datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .expect_err( "Shouldn't be able to add running instances to affinity groups", @@ -2134,11 +2131,7 @@ mod tests { // If we have no reservation for the instance, we can add it to the group. delete_instance_reservation(&datastore, instance).await; datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .unwrap(); @@ -2149,20 +2142,20 @@ mod tests { let members = datastore .affinity_group_member_list(&opctx, &authz_group, &pagbyid) .await - .unwrap(); + .unwrap() + .into_iter() + .map(|(m, _)| m.instance_id.into()) + .collect::>(); assert_eq!(members.len(), 1); - assert_eq!( - external::AffinityGroupMember::Instance(instance), - members[0].clone().into() - ); + assert_eq!(instance, members[0]); // We can delete the member and observe an empty member list -- even // though it's running! datastore - .affinity_group_member_delete( + .affinity_group_member_instance_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -2363,10 +2356,10 @@ mod tests { // Add the instance to the affinity group datastore - .affinity_group_member_add( + .affinity_group_member_instance_add( &opctx, &authz_a_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -2500,11 +2493,7 @@ mod tests { ) .await; datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .unwrap(); @@ -2772,11 +2761,7 @@ mod tests { ) .await; datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .unwrap(); @@ -2957,10 +2942,10 @@ mod tests { // Expect to see specific errors, depending on whether or not the // group/instance exist. let err = datastore - .affinity_group_member_add( + .affinity_group_member_instance_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .expect_err("Should have failed"); @@ -2989,10 +2974,10 @@ mod tests { // Do the same thing, but for group membership removal. let err = datastore - .affinity_group_member_delete( + .affinity_group_member_instance_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .expect_err("Should have failed"); @@ -3321,21 +3306,13 @@ mod tests { // Add the instance to the group datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .unwrap(); // Add the instance to the group again let err = datastore - .affinity_group_member_add( - &opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance), - ) + .affinity_group_member_instance_add(&opctx, &authz_group, instance) .await .unwrap_err(); assert!( @@ -3367,18 +3344,18 @@ mod tests { // We should be able to delete the membership idempotently. datastore - .affinity_group_member_delete( + .affinity_group_member_instance_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); let err = datastore - .affinity_group_member_delete( + .affinity_group_member_instance_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance), + instance, ) .await .unwrap_err(); diff --git a/nexus/db-queries/src/db/pub_test_utils/helpers.rs b/nexus/db-queries/src/db/pub_test_utils/helpers.rs index 88dc9c3047b..536ff4cabba 100644 --- a/nexus/db-queries/src/db/pub_test_utils/helpers.rs +++ b/nexus/db-queries/src/db/pub_test_utils/helpers.rs @@ -366,12 +366,8 @@ pub async fn create_affinity_group_member( .lookup_for(authz::Action::Modify) .await?; - db.affinity_group_member_add( - opctx, - &authz_group, - external::AffinityGroupMember::Instance(instance_id), - ) - .await?; + db.affinity_group_member_instance_add(opctx, &authz_group, instance_id) + .await?; Ok(()) } diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 59ba14e5fb8..f233a45b8d6 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -12,6 +12,9 @@ affinity_group_view GET /v1/affinity-groups/{affinity_ anti_affinity_group_create POST /v1/anti-affinity-groups anti_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group} anti_affinity_group_list GET /v1/anti-affinity-groups +anti_affinity_group_member_affinity_group_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} +anti_affinity_group_member_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} +anti_affinity_group_member_affinity_group_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} anti_affinity_group_member_instance_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} anti_affinity_group_member_instance_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} anti_affinity_group_member_instance_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 0930e74ece4..e9d381e5d8a 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -12,6 +12,7 @@ use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; use nexus_types::external_api::params; use nexus_types::external_api::views; +use nexus_types::identity::Resource; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; @@ -247,7 +248,7 @@ impl super::Nexus { .affinity_group_member_list(opctx, &authz_affinity_group, pagparams) .await? .into_iter() - .map(Into::into) + .map(|(member, name)| member.to_external(name.into())) .collect()) } @@ -279,12 +280,14 @@ impl super::Nexus { affinity_group_lookup.lookup_for(authz::Action::Read).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(authz_instance.id()), - ); + let member = InstanceUuid::from_untyped_uuid(authz_instance.id()); self.db_datastore - .affinity_group_member_view(opctx, &authz_affinity_group, member) + .affinity_group_member_instance_view( + opctx, + &authz_affinity_group, + member, + ) .await } @@ -342,20 +345,21 @@ impl super::Nexus { ) -> Result { let (.., authz_affinity_group) = affinity_group_lookup.lookup_for(authz::Action::Modify).await?; - let (.., authz_instance) = - instance_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(authz_instance.id()), - ); + let (.., authz_instance, instance) = + instance_lookup.fetch_for(authz::Action::Read).await?; + let member = InstanceUuid::from_untyped_uuid(authz_instance.id()); self.db_datastore - .affinity_group_member_add( + .affinity_group_member_instance_add( opctx, &authz_affinity_group, member.clone(), ) .await?; - Ok(member) + Ok(external::AffinityGroupMember::Instance { + id: member, + name: instance.name().clone(), + }) } pub(crate) async fn anti_affinity_group_member_instance_add( @@ -418,12 +422,14 @@ impl super::Nexus { affinity_group_lookup.lookup_for(authz::Action::Modify).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(authz_instance.id()), - ); + let member = InstanceUuid::from_untyped_uuid(authz_instance.id()); self.db_datastore - .affinity_group_member_delete(opctx, &authz_affinity_group, member) + .affinity_group_member_instance_delete( + opctx, + &authz_affinity_group, + member, + ) .await } diff --git a/openapi/nexus.json b/openapi/nexus.json index ec00fdab967..83da0d0428c 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -1487,12 +1487,160 @@ } } }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group member (where that member is an affinity group)", + "operationId": "anti_affinity_group_member_affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an anti-affinity group (where that member is an affinity group)", + "operationId": "anti_affinity_group_member_affinity_group_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an anti-affinity group (where that member is an affinity group)", + "operationId": "anti_affinity_group_member_affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { "get": { "tags": [ "affinity" ], - "summary": "Fetch an anti-affinity group member", + "summary": "Fetch an anti-affinity group member (where that member is an instance)", "operationId": "anti_affinity_group_member_instance_view", "parameters": [ { @@ -1543,7 +1691,7 @@ "tags": [ "affinity" ], - "summary": "Add a member to an anti-affinity group", + "summary": "Add a member to an anti-affinity group (where that member is an instance)", "operationId": "anti_affinity_group_member_instance_add", "parameters": [ { @@ -1594,7 +1742,7 @@ "tags": [ "affinity" ], - "summary": "Remove a member from an anti-affinity group", + "summary": "Remove a member from an anti-affinity group (where that member is an instance)", "operationId": "anti_affinity_group_member_instance_delete", "parameters": [ { @@ -12218,7 +12366,19 @@ ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/TypedUuidForInstanceKind" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "id", + "name" + ] } }, "required": [ @@ -12487,17 +12647,17 @@ "description": "A member of an Anti-Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { - "description": "An instance belonging to this group, identified by UUID.", + "description": "An affinity group belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "instance" + "affinity_group" ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" + "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" } }, "required": [ @@ -12506,17 +12666,17 @@ ] }, { - "description": "An affinity group belonging to this group, identified by UUID.", + "description": "An instance belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "affinity_group" + "instance" ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" + "$ref": "#/components/schemas/TypedUuidForInstanceKind" } }, "required": [ From 2f8a0e61c92cb3770a5ae437a33b2a06fbc3bc5d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 26 Feb 2025 16:49:24 -0800 Subject: [PATCH 72/84] AntiAffinityGroupMember exposes member name --- common/src/api/external/mod.rs | 17 +- nexus/db-model/src/affinity.rs | 29 +- nexus/db-queries/src/db/datastore/affinity.rs | 465 +++++++++--------- .../src/db/pub_test_utils/helpers.rs | 4 +- nexus/src/app/affinity.rs | 63 ++- openapi/nexus.json | 34 +- 6 files changed, 325 insertions(+), 287 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 229aa4776ac..1515e88db00 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1343,18 +1343,23 @@ impl SimpleIdentity for AffinityGroupMember { )] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { - /// An affinity group belonging to this group, identified by UUID. - AffinityGroup(AffinityGroupUuid), + /// An affinity group belonging to this group + AffinityGroup { id: AffinityGroupUuid, name: Name }, - /// An instance belonging to this group, identified by UUID. - Instance(InstanceUuid), + /// An instance belonging to this group + Instance { id: InstanceUuid, name: Name }, } +// TODO: revisit this impl too impl SimpleIdentity for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { - AntiAffinityGroupMember::AffinityGroup(id) => *id.as_untyped_uuid(), - AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), + AntiAffinityGroupMember::AffinityGroup { id, .. } => { + *id.as_untyped_uuid() + } + AntiAffinityGroupMember::Instance { id, .. } => { + *id.as_untyped_uuid() + } } } } diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index 4999a12dc4e..01063f249e9 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -253,13 +253,15 @@ impl AntiAffinityGroupInstanceMembership { ) -> Self { Self { group_id: group_id.into(), instance_id: instance_id.into() } } -} -impl From - for external::AntiAffinityGroupMember -{ - fn from(member: AntiAffinityGroupInstanceMembership) -> Self { - Self::Instance(member.instance_id.into()) + pub fn to_external( + self, + member_name: external::Name, + ) -> external::AntiAffinityGroupMember { + external::AntiAffinityGroupMember::Instance { + id: self.instance_id.into(), + name: member_name, + } } } @@ -280,12 +282,13 @@ impl AntiAffinityGroupAffinityMembership { affinity_group_id: affinity_group_id.into(), } } -} - -impl From - for external::AntiAffinityGroupMember -{ - fn from(member: AntiAffinityGroupAffinityMembership) -> Self { - Self::AffinityGroup(member.affinity_group_id.into()) + pub fn to_external( + self, + member_name: external::Name, + ) -> external::AntiAffinityGroupMember { + external::AntiAffinityGroupMember::AffinityGroup { + id: self.affinity_group_id.into(), + name: member_name, + } } } diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 3e562a77bed..cbf301815f5 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -421,19 +421,26 @@ impl DataStore { let mut query = QueryBuilder::new() .sql( - "SELECT id,label FROM ( - SELECT instance_id as id, 'instance' as label - FROM anti_affinity_group_instance_membership - WHERE group_id = ", + "SELECT id,name,label + FROM ( + SELECT instance_id as id, name, 'instance' as label + FROM anti_affinity_group_instance_membership + INNER JOIN instance + ON instance.id = anti_affinity_group_instance_membership.instance_id + WHERE instance.time_deleted IS NULL AND + group_id = ", ) .param() .bind::(authz_anti_affinity_group.id()) .sql( " UNION - SELECT affinity_group_id as id, 'affinity_group' as label - FROM anti_affinity_group_affinity_membership - WHERE anti_affinity_group_id = ", + SELECT affinity_group_id as id, name, 'affinity_group' as label + FROM anti_affinity_group_affinity_membership + INNER JOIN affinity_group + ON affinity_group.id = anti_affinity_group_affinity_membership.affinity_group_id + WHERE affinity_group.time_deleted IS NULL AND + anti_affinity_group_id = ", ) .param() .bind::(authz_anti_affinity_group.id()) @@ -460,22 +467,28 @@ impl DataStore { ); query - .query::<(diesel::sql_types::Uuid, diesel::sql_types::Text)>() - .load_async::<(Uuid, String)>( + .query::<( + diesel::sql_types::Uuid, + diesel::sql_types::Text, + diesel::sql_types::Text, + )>() + .load_async::<(Uuid, Name, String)>( &*self.pool_connection_authorized(opctx).await?, ) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .into_iter() - .map(|(id, label)| { + .map(|(id, name, label)| { use external::AntiAffinityGroupMember as Member; match label.as_str() { - "affinity_group" => Ok(Member::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(id), - )), - "instance" => Ok(Member::Instance( - InstanceUuid::from_untyped_uuid(id), - )), + "affinity_group" => Ok(Member::AffinityGroup { + id: AffinityGroupUuid::from_untyped_uuid(id), + name: name.into(), + }), + "instance" => Ok(Member::Instance { + id: InstanceUuid::from_untyped_uuid(id), + name: name.into(), + }), other => Err(external::Error::internal_error(&format!( "Unexpected label from database query: {other}" ))), @@ -521,69 +534,88 @@ impl DataStore { }) } - pub async fn anti_affinity_group_member_view( + pub async fn anti_affinity_group_member_instance_view( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, - member: external::AntiAffinityGroupMember, + instance_id: InstanceUuid, ) -> Result { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; let conn = self.pool_connection_authorized(opctx).await?; - match member { - external::AntiAffinityGroupMember::Instance(instance_id) => { - use db::schema::anti_affinity_group_instance_membership::dsl; - dsl::anti_affinity_group_instance_membership - .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter( - dsl::instance_id.eq(instance_id.into_untyped_uuid()), - ) - .select(AntiAffinityGroupInstanceMembership::as_select()) - .get_result_async(&*conn) - .await - .map(|m| m.into()) - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::AntiAffinityGroupMember, - LookupType::by_id( - instance_id.into_untyped_uuid(), - ), - ), - ) - }) - } - external::AntiAffinityGroupMember::AffinityGroup( - affinity_group_id, - ) => { - use db::schema::anti_affinity_group_affinity_membership::dsl; - dsl::anti_affinity_group_affinity_membership - .filter( - dsl::anti_affinity_group_id - .eq(authz_anti_affinity_group.id()), - ) - .filter( - dsl::affinity_group_id - .eq(affinity_group_id.into_untyped_uuid()), - ) - .select(AntiAffinityGroupAffinityMembership::as_select()) - .get_result_async(&*conn) - .await - .map(|m| m.into()) - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::AntiAffinityGroupMember, - LookupType::by_id( - affinity_group_id.into_untyped_uuid(), - ), - ), - ) - }) - } - } + use db::schema::anti_affinity_group_instance_membership::dsl; + use db::schema::instance::dsl as instance_dsl; + dsl::anti_affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .inner_join( + instance_dsl::instance + .on(instance_dsl::id.eq(dsl::instance_id)), + ) + .filter(instance_dsl::time_deleted.is_null()) + .select(( + AntiAffinityGroupInstanceMembership::as_select(), + instance_dsl::name, + )) + .get_result_async::<(AntiAffinityGroupInstanceMembership, Name)>( + &*conn, + ) + .await + .map(|(member, name)| member.to_external(name.into())) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id(instance_id.into_untyped_uuid()), + ), + ) + }) + } + + pub async fn anti_affinity_group_member_affinity_group_view( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + affinity_group_id: AffinityGroupUuid, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + use db::schema::affinity_group::dsl as affinity_group_dsl; + use db::schema::anti_affinity_group_affinity_membership::dsl; + dsl::anti_affinity_group_affinity_membership + .filter( + dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id()), + ) + .filter( + dsl::affinity_group_id + .eq(affinity_group_id.into_untyped_uuid()), + ) + .inner_join( + affinity_group_dsl::affinity_group + .on(affinity_group_dsl::id.eq(dsl::affinity_group_id)), + ) + .select(( + AntiAffinityGroupAffinityMembership::as_select(), + affinity_group_dsl::name, + )) + .get_result_async::<(AntiAffinityGroupAffinityMembership, Name)>( + &*conn, + ) + .await + .map(|(member, name)| member.to_external(name.into())) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id( + affinity_group_id.into_untyped_uuid(), + ), + ), + ) + }) } pub async fn affinity_group_member_instance_add( @@ -709,45 +741,19 @@ impl DataStore { Ok(()) } - pub async fn anti_affinity_group_member_add( + pub async fn anti_affinity_group_member_instance_add( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, - member: external::AntiAffinityGroupMember, + instance_id: InstanceUuid, ) -> Result<(), Error> { opctx .authorize(authz::Action::Modify, authz_anti_affinity_group) .await?; - match member { - external::AntiAffinityGroupMember::Instance(id) => { - self.anti_affinity_group_member_add_instance( - opctx, - authz_anti_affinity_group, - id, - ) - .await - } - external::AntiAffinityGroupMember::AffinityGroup(id) => { - self.anti_affinity_group_member_add_group( - opctx, - authz_anti_affinity_group, - id, - ) - .await - } - } - } - - async fn anti_affinity_group_member_add_instance( - &self, - opctx: &OpContext, - authz_anti_affinity_group: &authz::AntiAffinityGroup, - instance_id: InstanceUuid, - ) -> Result<(), Error> { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_add_instance") + self.transaction_retry_wrapper("anti_affinity_group_member_instance_add") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -851,15 +857,19 @@ impl DataStore { Ok(()) } - async fn anti_affinity_group_member_add_group( + pub async fn anti_affinity_group_member_affinity_group_add( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, affinity_group_id: AffinityGroupUuid, ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_add_group") + self.transaction_retry_wrapper("anti_affinity_group_member_affinity_group_add") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as anti_affinity_group_dsl; @@ -1067,48 +1077,20 @@ impl DataStore { Ok(()) } - pub async fn anti_affinity_group_member_delete( + /// Deletes an anti-affinity member, when that member is an instance + pub async fn anti_affinity_group_member_instance_delete( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, - member: external::AntiAffinityGroupMember, + instance_id: InstanceUuid, ) -> Result<(), Error> { opctx .authorize(authz::Action::Modify, authz_anti_affinity_group) .await?; - match member { - external::AntiAffinityGroupMember::Instance(id) => { - self.anti_affinity_group_instance_member_delete( - opctx, - authz_anti_affinity_group, - id, - ) - .await - } - external::AntiAffinityGroupMember::AffinityGroup(id) => { - self.anti_affinity_group_affinity_member_delete( - opctx, - authz_anti_affinity_group, - id, - ) - .await - } - } - } - - // Deletes an anti-affinity member, when that member is an instance - // - // See: [`Self::anti_affinity_group_member_delete`] - async fn anti_affinity_group_instance_member_delete( - &self, - opctx: &OpContext, - authz_anti_affinity_group: &authz::AntiAffinityGroup, - instance_id: InstanceUuid, - ) -> Result<(), Error> { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_instance_member_delete") + self.transaction_retry_wrapper("anti_affinity_group_member_instance_delete") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -1161,18 +1143,20 @@ impl DataStore { Ok(()) } - // Deletes an anti-affinity member, when that member is an affinity group - // - // See: [`Self::anti_affinity_group_member_delete`] - async fn anti_affinity_group_affinity_member_delete( + /// Deletes an anti-affinity member, when that member is an affinity group + pub async fn anti_affinity_group_member_affinity_group_delete( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, affinity_group_id: AffinityGroupUuid, ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_affinity_member_delete") + self.transaction_retry_wrapper("anti_affinity_group_member_affinity_group_delete") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -1857,10 +1841,10 @@ mod tests { // Add the instance as a member to the group datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -1871,17 +1855,20 @@ mod tests { .await .unwrap(); assert_eq!(members.len(), 1); - assert_eq!( - external::AntiAffinityGroupMember::Instance(instance), - members[0].clone() - ); + assert!(matches!( + members[0], + external::AntiAffinityGroupMember::Instance { + id, + .. + } if id == instance, + )); // We can delete the member and observe an empty member list datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -1951,21 +1938,25 @@ mod tests { let mut members = Vec::new(); for i in 0..INSTANCE_COUNT { + let name = format!("instance-{i}"); let instance = create_stopped_instance_record( &opctx, &datastore, &authz_project, - &format!("instance-{i}"), + &name, ) .await; // Add the instance as a member to the group - let member = external::AntiAffinityGroupMember::Instance(instance); + let member = external::AntiAffinityGroupMember::Instance { + id: instance, + name: name.try_into().unwrap(), + }; datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_aa_group, - member.clone(), + instance, ) .await .unwrap(); @@ -1973,24 +1964,28 @@ mod tests { } for i in 0..AFFINITY_GROUP_COUNT { + let name = format!("affinity-{i}"); let affinity_group = create_affinity_group( &opctx, &datastore, &authz_project, - &format!("affinity-{i}"), + &name, ) .await .unwrap(); + let affinity_group_id = + AffinityGroupUuid::from_untyped_uuid(affinity_group.id()); // Add the instance as a member to the group - let member = external::AntiAffinityGroupMember::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), - ); + let member = external::AntiAffinityGroupMember::AffinityGroup { + id: affinity_group_id, + name: name.try_into().unwrap(), + }; datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_affinity_group_add( &opctx, &authz_aa_group, - member.clone(), + affinity_group_id, ) .await .unwrap(); @@ -2221,10 +2216,10 @@ mod tests { // Cannot add the instance to the group while it's running. let err = datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .expect_err( @@ -2241,10 +2236,10 @@ mod tests { // If we have no reservation for the instance, we can add it to the group. delete_instance_reservation(&datastore, instance).await; datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -2258,18 +2253,20 @@ mod tests { .await .unwrap(); assert_eq!(members.len(), 1); - assert_eq!( - external::AntiAffinityGroupMember::Instance(instance), - members[0].clone() - ); - + assert!(matches!( + members[0], + external::AntiAffinityGroupMember::Instance { + id, + .. + } if id == instance, + )); // We can delete the member and observe an empty member list -- even // though it's running! datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -2369,10 +2366,10 @@ mod tests { // Now, we cannot add the affinity group to the anti-affinity group let err = datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_affinity_group_add( &opctx, &authz_aa_group, - external::AntiAffinityGroupMember::AffinityGroup(affinity_group_id), + affinity_group_id, ) .await .expect_err( @@ -2390,12 +2387,10 @@ mod tests { // anti-affinity group. delete_instance_reservation(&datastore, instance).await; datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_affinity_group_add( &opctx, &authz_aa_group, - external::AntiAffinityGroupMember::AffinityGroup( - affinity_group_id, - ), + affinity_group_id, ) .await .unwrap(); @@ -2413,20 +2408,21 @@ mod tests { .await .unwrap(); assert_eq!(members.len(), 1); - assert_eq!( - external::AntiAffinityGroupMember::AffinityGroup(affinity_group_id), - members[0].clone() - ); + assert!(matches!( + members[0], + external::AntiAffinityGroupMember::AffinityGroup { + id, + .. + } if id == affinity_group_id, + )); // We can delete the member and observe an empty member list -- even // though it has an instance which is running! datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_affinity_group_delete( &opctx, &authz_aa_group, - external::AntiAffinityGroupMember::AffinityGroup( - affinity_group_id, - ), + affinity_group_id, ) .await .unwrap(); @@ -2573,21 +2569,19 @@ mod tests { ) .await; datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_aa_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); // Also, add the affinity member to the anti-affinity group as a member datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_affinity_group_add( &opctx, &authz_aa_group, - external::AntiAffinityGroupMember::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), - ), + AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), ) .await .unwrap(); @@ -2664,24 +2658,22 @@ mod tests { .unwrap(); // Add the affinity group to the anti-affinity group. - let member = external::AntiAffinityGroupMember::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), - ); + let member = AffinityGroupUuid::from_untyped_uuid(affinity_group.id()); datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_affinity_group_add( &opctx, &authz_aa_group, - member.clone(), + member, ) .await .unwrap(); // Right now, the affinity group is observable datastore - .anti_affinity_group_member_view( + .anti_affinity_group_member_affinity_group_view( &opctx, &authz_aa_group, - member.clone(), + member, ) .await .expect("Group member should be visible - we just added it"); @@ -2691,10 +2683,10 @@ mod tests { // The affinity group membership should have been revoked let err = datastore - .anti_affinity_group_member_view( + .anti_affinity_group_member_affinity_group_view( &opctx, &authz_aa_group, - member.clone(), + member, ) .await .expect_err("Group member should no longer exist"); @@ -2836,10 +2828,10 @@ mod tests { ) .await; datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); @@ -3174,26 +3166,26 @@ mod tests { // // Expect to see specific errors, depending on whether or not the // group/member exist. - let member = match arg.member_type { - Member::Instance => { - external::AntiAffinityGroupMember::Instance(instance) - } - Member::AffinityGroup => { - external::AntiAffinityGroupMember::AffinityGroup( + let err = match arg.member_type { + Member::Instance => datastore + .anti_affinity_group_member_instance_add( + &opctx, + &authz_group, + instance, + ) + .await + .expect_err("Should have failed"), + Member::AffinityGroup => datastore + .anti_affinity_group_member_affinity_group_add( + &opctx, + &authz_group, AffinityGroupUuid::from_untyped_uuid( affinity_group.id(), ), ) - } + .await + .expect_err("Should have failed"), }; - let err = datastore - .anti_affinity_group_member_add( - &opctx, - &authz_group, - member.clone(), - ) - .await - .expect_err("Should have failed"); match (arg.group, arg.member) { (false, _) => { @@ -3218,10 +3210,27 @@ mod tests { } // Do the same thing, but for group membership removal. - let err = datastore - .anti_affinity_group_member_delete(&opctx, &authz_group, member) - .await - .expect_err("Should have failed"); + let err = match arg.member_type { + Member::Instance => datastore + .anti_affinity_group_member_instance_delete( + &opctx, + &authz_group, + instance, + ) + .await + .expect_err("Should have failed"), + Member::AffinityGroup => datastore + .anti_affinity_group_member_affinity_group_delete( + &opctx, + &authz_group, + AffinityGroupUuid::from_untyped_uuid( + affinity_group.id(), + ), + ) + .await + .expect_err("Should have failed"), + }; + match (arg.group, arg.member) { (false, _) => { assert!( @@ -3417,20 +3426,20 @@ mod tests { // Add the instance to the group datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); // Add the instance to the group again let err = datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap_err(); @@ -3447,7 +3456,7 @@ mod tests { // We should still only observe a single member in the group. // - // Two calls to "anti_affinity_group_member_add" should be the same + // Two calls to "anti_affinity_group_member_instance_add" should be the same // as a single call. let pagparams = DataPageParams { marker: None, @@ -3462,18 +3471,18 @@ mod tests { // We should be able to delete the membership idempotently. datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap(); let err = datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance), + instance, ) .await .unwrap_err(); diff --git a/nexus/db-queries/src/db/pub_test_utils/helpers.rs b/nexus/db-queries/src/db/pub_test_utils/helpers.rs index 536ff4cabba..1b1f48bc287 100644 --- a/nexus/db-queries/src/db/pub_test_utils/helpers.rs +++ b/nexus/db-queries/src/db/pub_test_utils/helpers.rs @@ -386,10 +386,10 @@ pub async fn create_anti_affinity_group_member( .lookup_for(authz::Action::Modify) .await?; - db.anti_affinity_group_member_add( + db.anti_affinity_group_member_instance_add( opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance_id), + instance_id, ) .await?; Ok(()) diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index e9d381e5d8a..33559407336 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -301,12 +301,10 @@ impl super::Nexus { anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(authz_instance.id()), - ); + let member = InstanceUuid::from_untyped_uuid(authz_instance.id()); self.db_datastore - .anti_affinity_group_member_view( + .anti_affinity_group_member_instance_view( opctx, &authz_anti_affinity_group, member, @@ -324,12 +322,11 @@ impl super::Nexus { anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; let (.., authz_affinity_group) = affinity_group_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AntiAffinityGroupMember::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), - ); + let member = + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()); self.db_datastore - .anti_affinity_group_member_view( + .anti_affinity_group_member_affinity_group_view( opctx, &authz_anti_affinity_group, member, @@ -353,7 +350,7 @@ impl super::Nexus { .affinity_group_member_instance_add( opctx, &authz_affinity_group, - member.clone(), + member, ) .await?; Ok(external::AffinityGroupMember::Instance { @@ -371,20 +368,21 @@ impl super::Nexus { let (.., authz_anti_affinity_group) = anti_affinity_group_lookup .lookup_for(authz::Action::Modify) .await?; - let (.., authz_instance) = - instance_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(authz_instance.id()), - ); + let (.., authz_instance, instance) = + instance_lookup.fetch_for(authz::Action::Read).await?; + let member = InstanceUuid::from_untyped_uuid(authz_instance.id()); self.db_datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( opctx, &authz_anti_affinity_group, - member.clone(), + member, ) .await?; - Ok(member) + Ok(external::AntiAffinityGroupMember::Instance { + id: member, + name: instance.name().clone(), + }) } pub(crate) async fn anti_affinity_group_member_affinity_group_add( @@ -396,20 +394,22 @@ impl super::Nexus { let (.., authz_anti_affinity_group) = anti_affinity_group_lookup .lookup_for(authz::Action::Modify) .await?; - let (.., authz_affinity_group) = - affinity_group_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AntiAffinityGroupMember::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), - ); + let (.., authz_affinity_group, affinity_group) = + affinity_group_lookup.fetch_for(authz::Action::Read).await?; + let member = + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()); self.db_datastore - .anti_affinity_group_member_add( + .anti_affinity_group_member_affinity_group_add( opctx, &authz_anti_affinity_group, - member.clone(), + member, ) .await?; - Ok(member) + Ok(external::AntiAffinityGroupMember::AffinityGroup { + id: member, + name: affinity_group.name().clone(), + }) } pub(crate) async fn affinity_group_member_delete( @@ -444,12 +444,10 @@ impl super::Nexus { .await?; let (.., authz_instance) = instance_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AntiAffinityGroupMember::Instance( - InstanceUuid::from_untyped_uuid(authz_instance.id()), - ); + let member = InstanceUuid::from_untyped_uuid(authz_instance.id()); self.db_datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( opctx, &authz_anti_affinity_group, member, @@ -468,12 +466,11 @@ impl super::Nexus { .await?; let (.., authz_affinity_group) = affinity_group_lookup.lookup_for(authz::Action::Read).await?; - let member = external::AntiAffinityGroupMember::AffinityGroup( - AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), - ); + let member = + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()); self.db_datastore - .anti_affinity_group_member_delete( + .anti_affinity_group_member_affinity_group_delete( opctx, &authz_anti_affinity_group, member, diff --git a/openapi/nexus.json b/openapi/nexus.json index 83da0d0428c..38214bf4ca0 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12356,7 +12356,7 @@ "description": "A member of an Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { - "description": "An instance belonging to this group, identified by UUID.", + "description": "An instance belonging to this group", "type": "object", "properties": { "type": { @@ -12647,7 +12647,7 @@ "description": "A member of an Anti-Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { - "description": "An affinity group belonging to this group, identified by UUID.", + "description": "An affinity group belonging to this group", "type": "object", "properties": { "type": { @@ -12657,7 +12657,19 @@ ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "id", + "name" + ] } }, "required": [ @@ -12666,7 +12678,7 @@ ] }, { - "description": "An instance belonging to this group, identified by UUID.", + "description": "An instance belonging to this group", "type": "object", "properties": { "type": { @@ -12676,7 +12688,19 @@ ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/TypedUuidForInstanceKind" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "id", + "name" + ] } }, "required": [ From 388b6cd3dea17cf1e95c8f3e77946fd4c40c6f1c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 26 Feb 2025 16:51:27 -0800 Subject: [PATCH 73/84] Update API --- nexus/external-api/output/nexus_tags.txt | 3 + openapi/nexus.json | 166 +++++++++++++++++++++-- 2 files changed, 160 insertions(+), 9 deletions(-) diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 59ba14e5fb8..f233a45b8d6 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -12,6 +12,9 @@ affinity_group_view GET /v1/affinity-groups/{affinity_ anti_affinity_group_create POST /v1/anti-affinity-groups anti_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group} anti_affinity_group_list GET /v1/anti-affinity-groups +anti_affinity_group_member_affinity_group_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} +anti_affinity_group_member_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} +anti_affinity_group_member_affinity_group_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} anti_affinity_group_member_instance_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} anti_affinity_group_member_instance_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} anti_affinity_group_member_instance_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} diff --git a/openapi/nexus.json b/openapi/nexus.json index ec00fdab967..e69b7cff1bd 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -1487,12 +1487,160 @@ } } }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group member (where that member is an affinity group)", + "operationId": "anti_affinity_group_member_affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an anti-affinity group (where that member is an affinity group)", + "operationId": "anti_affinity_group_member_affinity_group_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an anti-affinity group (where that member is an affinity group)", + "operationId": "anti_affinity_group_member_affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { "get": { "tags": [ "affinity" ], - "summary": "Fetch an anti-affinity group member", + "summary": "Fetch an anti-affinity group member (where that member is an instance)", "operationId": "anti_affinity_group_member_instance_view", "parameters": [ { @@ -1543,7 +1691,7 @@ "tags": [ "affinity" ], - "summary": "Add a member to an anti-affinity group", + "summary": "Add a member to an anti-affinity group (where that member is an instance)", "operationId": "anti_affinity_group_member_instance_add", "parameters": [ { @@ -1594,7 +1742,7 @@ "tags": [ "affinity" ], - "summary": "Remove a member from an anti-affinity group", + "summary": "Remove a member from an anti-affinity group (where that member is an instance)", "operationId": "anti_affinity_group_member_instance_delete", "parameters": [ { @@ -12487,17 +12635,17 @@ "description": "A member of an Anti-Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { - "description": "An instance belonging to this group, identified by UUID.", + "description": "An affinity group belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "instance" + "affinity_group" ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" + "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" } }, "required": [ @@ -12506,17 +12654,17 @@ ] }, { - "description": "An affinity group belonging to this group, identified by UUID.", + "description": "An instance belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "affinity_group" + "instance" ] }, "value": { - "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" + "$ref": "#/components/schemas/TypedUuidForInstanceKind" } }, "required": [ From aa9de032193c1a35cc502921750557fc4c92367c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 27 Feb 2025 12:29:29 -0800 Subject: [PATCH 74/84] Unauthorized coverage test --- nexus/tests/integration_tests/endpoints.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 618752227e6..6c2dc7b8db3 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -516,6 +516,17 @@ pub static DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL: LazyLock = *DEMO_PROJECT_SELECTOR ) }); +pub static DEMO_ANTI_AFFINITY_GROUP_AFFINITY_GROUP_MEMBER_URL: LazyLock< + String, +> = LazyLock::new(|| { + format!( + "/v1/anti-affinity-groups/{}/members/affinity-group/{}?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, + *DEMO_AFFINITY_GROUP_NAME, + *DEMO_PROJECT_SELECTOR + ) +}); + pub static DEMO_ANTI_AFFINITY_GROUP_CREATE: LazyLock< params::AntiAffinityGroupCreate, > = LazyLock::new(|| params::AntiAffinityGroupCreate { @@ -1994,6 +2005,17 @@ pub static VERIFY_ENDPOINTS: LazyLock> = AllowedMethod::Post(serde_json::Value::Null), ], }, + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_AFFINITY_GROUP_MEMBER_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Post(serde_json::Value::Null), + ], + }, VerifyEndpoint { url: &DEMO_IMPORT_DISK_BULK_WRITE_START_URL, visibility: Visibility::Protected, From 20496ce0aa0958b38aa2544caad6f60b1e1ba1c2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 27 Feb 2025 17:04:03 -0800 Subject: [PATCH 75/84] i scream you scream we all scream for authcream --- nexus/tests/integration_tests/unauthorized.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index 1f45597ce74..d6802f06b8c 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -304,7 +304,7 @@ static SETUP_REQUESTS: LazyLock> = LazyLock::new(|| { body: serde_json::to_value(&*DEMO_AFFINITY_GROUP_CREATE).unwrap(), id_routes: vec!["/v1/affinity-groups/{id}"], }, - // Add a member to the affinity group + // Add an instance member to the affinity group SetupReq::Post { url: &DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL, body: serde_json::Value::Null, @@ -317,12 +317,18 @@ static SETUP_REQUESTS: LazyLock> = LazyLock::new(|| { .unwrap(), id_routes: vec!["/v1/anti-affinity-groups/{id}"], }, - // Add a member to the anti-affinity group + // Add an instance member to the anti-affinity group SetupReq::Post { url: &DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL, body: serde_json::Value::Null, id_routes: vec![], }, + // Add an affinity group member to the affinity group + SetupReq::Post { + url: &DEMO_ANTI_AFFINITY_GROUP_AFFINITY_GROUP_MEMBER_URL, + body: serde_json::Value::Null, + id_routes: vec![], + }, // Lookup the previously created NIC SetupReq::Get { url: &DEMO_INSTANCE_NIC_URL, From be6ac0faddb39e8cfd8712f0b2283ca30e2df50a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 28 Feb 2025 14:07:11 -0800 Subject: [PATCH 76/84] Name-based pagination (needs tests) --- common/src/api/external/mod.rs | 19 +- nexus/db-queries/src/db/datastore/affinity.rs | 223 ++++++++++++------ nexus/external-api/src/lib.rs | 4 +- nexus/src/app/affinity.rs | 7 +- nexus/src/external_api/http_entrypoints.rs | 22 +- 5 files changed, 179 insertions(+), 96 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 1515e88db00..17f49f47524 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1325,13 +1325,18 @@ pub enum AffinityGroupMember { Instance { id: InstanceUuid, name: Name }, } -// TODO: Revisit this impl.. it was originally created for UUID-only members -impl SimpleIdentity for AffinityGroupMember { +impl SimpleIdentityOrName for AffinityGroupMember { fn id(&self) -> Uuid { match self { AffinityGroupMember::Instance { id, .. } => *id.as_untyped_uuid(), } } + + fn name(&self) -> &Name { + match self { + AffinityGroupMember::Instance { name, .. } => name, + } + } } /// A member of an Anti-Affinity Group @@ -1350,8 +1355,7 @@ pub enum AntiAffinityGroupMember { Instance { id: InstanceUuid, name: Name }, } -// TODO: revisit this impl too -impl SimpleIdentity for AntiAffinityGroupMember { +impl SimpleIdentityOrName for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { AntiAffinityGroupMember::AffinityGroup { id, .. } => { @@ -1362,6 +1366,13 @@ impl SimpleIdentity for AntiAffinityGroupMember { } } } + + fn name(&self) -> &Name { + match self { + AntiAffinityGroupMember::AffinityGroup { name, .. } => name, + AntiAffinityGroupMember::Instance { name, .. } => name, + } + } } // DISKS diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index cbf301815f5..3c18a0db77f 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -375,46 +375,99 @@ impl DataStore { opctx: &OpContext, authz_affinity_group: &authz::AffinityGroup, pagparams: &PaginatedBy<'_>, - ) -> ListResultVec<(AffinityGroupInstanceMembership, Name)> { + ) -> ListResultVec { opctx.authorize(authz::Action::Read, authz_affinity_group).await?; - use db::schema::affinity_group_instance_membership::dsl; - use db::schema::instance::dsl as instance_dsl; + let mut query = QueryBuilder::new() + .sql(" + SELECT instance_id as id, name + FROM affinity_group_instance_membership + INNER JOIN instance + ON instance.id = affinity_group_instance_membership.instance_id + WHERE instance.time_deleted IS NULL AND group_id = ", + ) + .param() + .bind::(authz_affinity_group.id()) + .sql(" "); + + let (direction, limit) = match pagparams { + PaginatedBy::Id(p) => (p.direction, p.limit), + PaginatedBy::Name(p) => (p.direction, p.limit), + }; + let asc = match direction { + dropshot::PaginationOrder::Ascending => true, + dropshot::PaginationOrder::Descending => false, + }; + match pagparams { - PaginatedBy::Id(pagparams) => paginated( - dsl::affinity_group_instance_membership, - dsl::instance_id, - &pagparams, - ), - PaginatedBy::Name(_) => { - return Err(Error::invalid_request( - "Cannot paginate group members by name", - )); - } + PaginatedBy::Id(DataPageParams { marker, .. }) => { + if let Some(id) = marker { + query = query + .sql("WHERE id ") + .sql(if asc { ">" } else { "<" }) + .sql(" ") + .param() + .bind::(**id); + }; + query = query.sql(" ORDER BY id "); + }, + PaginatedBy::Name(DataPageParams { marker, .. }) => { + if let Some(name) = marker { + query = query + .sql("WHERE name ") + .sql(if asc { ">" } else { "<" }) + .sql(" ") + .param() + .bind::(Name((*name).clone())); + }; + query = query.sql(" ORDER BY name "); + + }, } - .filter(dsl::group_id.eq(authz_affinity_group.id())) - .inner_join( - instance_dsl::instance.on(instance_dsl::id.eq(dsl::instance_id)), - ) - .filter(instance_dsl::time_deleted.is_null()) - .select(( - AffinityGroupInstanceMembership::as_select(), - instance_dsl::name, - )) - .load_async(&*self.pool_connection_authorized(opctx).await?) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + if asc { + query = query.sql("ASC "); + } else { + query = query.sql("DESC "); + } + + query = + query.sql(" LIMIT ").param().bind::( + i64::from(limit.get()), + ); + + query + .query::<( + diesel::sql_types::Uuid, + diesel::sql_types::Text, + )>() + .load_async::<(Uuid, Name)>( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|(id, name)| { + Ok(external::AffinityGroupMember::Instance { + id: InstanceUuid::from_untyped_uuid(id), + name: name.into(), + }) + }) + .collect() } pub async fn anti_affinity_group_member_list( &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, - pagparams: &DataPageParams<'_, Uuid>, + pagparams: &PaginatedBy<'_>, ) -> ListResultVec { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; - let asc = match pagparams.direction { + let (direction, limit) = match pagparams { + PaginatedBy::Id(p) => (p.direction, p.limit), + PaginatedBy::Name(p) => (p.direction, p.limit), + }; + let asc = match direction { dropshot::PaginationOrder::Ascending => true, dropshot::PaginationOrder::Descending => false, }; @@ -445,16 +498,32 @@ impl DataStore { .param() .bind::(authz_anti_affinity_group.id()) .sql(") "); - if let Some(id) = pagparams.marker { - query = query - .sql("WHERE id ") - .sql(if asc { ">" } else { "<" }) - .sql(" ") - .param() - .bind::(*id); - }; - query = query.sql(" ORDER BY id "); + match pagparams { + PaginatedBy::Id(DataPageParams { marker, .. }) => { + if let Some(id) = marker { + query = query + .sql("WHERE id ") + .sql(if asc { ">" } else { "<" }) + .sql(" ") + .param() + .bind::(**id); + }; + query = query.sql(" ORDER BY id "); + }, + PaginatedBy::Name(DataPageParams { marker, .. }) => { + if let Some(name) = marker { + query = query + .sql("WHERE name ") + .sql(if asc { ">" } else { "<" }) + .sql(" ") + .param() + .bind::(Name((*name).clone())); + }; + query = query.sql(" ORDER BY name "); + + }, + } if asc { query = query.sql("ASC "); } else { @@ -463,7 +532,7 @@ impl DataStore { query = query.sql(" LIMIT ").param().bind::( - i64::from(pagparams.limit.get()), + i64::from(limit.get()), ); query @@ -1223,7 +1292,7 @@ mod tests { use nexus_types::external_api::params; use omicron_common::api::external::{ self, ByteCount, DataPageParams, IdentityMetadataCreateParams, - SimpleIdentity, + SimpleIdentityOrName, }; use omicron_test_utils::dev; use omicron_uuid_kinds::GenericUuid; @@ -1766,7 +1835,7 @@ mod tests { .await .unwrap() .into_iter() - .map(|(m, _)| m.instance_id.into()) + .map(|m| InstanceUuid::from_untyped_uuid(m.id())) .collect::>(); assert_eq!(members.len(), 1); assert_eq!(instance, members[0]); @@ -1819,11 +1888,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await @@ -1915,11 +1984,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list( &opctx, @@ -1996,11 +2065,11 @@ mod tests { members.sort_unstable_by_key(|m1| m1.id()); // We can list all members - let mut pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let observed_members = datastore .anti_affinity_group_member_list( &opctx, @@ -2013,7 +2082,12 @@ mod tests { // We can paginate over the results let marker = members[2].id(); - pagparams.marker = Some(&marker); + let pagparams = PaginatedBy::Id(DataPageParams { + marker: Some(&marker), + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + let observed_members = datastore .anti_affinity_group_member_list( &opctx, @@ -2025,8 +2099,11 @@ mod tests { assert_eq!(observed_members, members[3..]); // We can list limited results - pagparams.marker = Some(&marker); - pagparams.limit = NonZeroU32::new(2).unwrap(); + let pagparams = PaginatedBy::Id(DataPageParams { + marker: Some(&marker), + limit: NonZeroU32::new(2).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); let observed_members = datastore .anti_affinity_group_member_list( &opctx, @@ -2039,9 +2116,11 @@ mod tests { // We can list in descending order too members.reverse(); - pagparams.marker = None; - pagparams.limit = NonZeroU32::new(100).unwrap(); - pagparams.direction = dropshot::PaginationOrder::Descending; + let pagparams = PaginatedBy::Id(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Descending + }); let observed_members = datastore .anti_affinity_group_member_list( &opctx, @@ -2085,12 +2164,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagbyid = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; - let pagbyid = PaginatedBy::Id(pagparams_id); + }); let members = datastore .affinity_group_member_list(&opctx, &authz_group, &pagbyid) .await @@ -2139,7 +2217,7 @@ mod tests { .await .unwrap() .into_iter() - .map(|(m, _)| m.instance_id.into()) + .map(|m| InstanceUuid::from_untyped_uuid(m.id())) .collect::>(); assert_eq!(members.len(), 1); assert_eq!(instance, members[0]); @@ -2193,11 +2271,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await @@ -2327,11 +2405,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list( &opctx, @@ -2468,12 +2546,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagbyid = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; - let pagbyid = PaginatedBy::Id(pagparams_id); + }); let members = datastore .affinity_group_member_list(&opctx, &authz_group, &pagbyid) .await @@ -2545,11 +2622,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list( &opctx, @@ -2732,12 +2809,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagbyid = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; - let pagbyid = PaginatedBy::Id(pagparams_id); + }); let members = datastore .affinity_group_member_list(&opctx, &authz_group, &pagbyid) .await @@ -2808,11 +2884,11 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await @@ -3339,12 +3415,11 @@ mod tests { // // Two calls to "affinity_group_member_add" should be the same // as a single call. - let pagparams_id = DataPageParams { + let pagbyid = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; - let pagbyid = PaginatedBy::Id(pagparams_id); + }); let members = datastore .affinity_group_member_list(&opctx, &authz_group, &pagbyid) .await @@ -3458,11 +3533,11 @@ mod tests { // // Two calls to "anti_affinity_group_member_instance_add" should be the same // as a single call. - let pagparams = DataPageParams { + let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, - }; + }); let members = datastore .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index b03e717e1df..55678a1b22c 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -1297,7 +1297,7 @@ pub trait NexusExternalApi { }] async fn affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query>, path_params: Path, ) -> Result>, HttpError>; @@ -1405,7 +1405,7 @@ pub trait NexusExternalApi { }] async fn anti_affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query>, path_params: Path, ) -> Result>, HttpError>; diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 33559407336..60384a5d804 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -16,7 +16,6 @@ use nexus_types::identity::Resource; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; -use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; @@ -247,16 +246,14 @@ impl super::Nexus { .db_datastore .affinity_group_member_list(opctx, &authz_affinity_group, pagparams) .await? - .into_iter() - .map(|(member, name)| member.to_external(name.into())) - .collect()) + ) } pub(crate) async fn anti_affinity_group_member_list( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, - pagparams: &DataPageParams<'_, uuid::Uuid>, + pagparams: &PaginatedBy<'_>, ) -> ListResultVec { let (.., authz_anti_affinity_group) = anti_affinity_group_lookup .lookup_for(authz::Action::ListChildren) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 246cc86c551..ef7dab2d739 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -54,7 +54,6 @@ use nexus_types::{ }, }; use omicron_common::api::external::http_pagination::data_page_params_for; -use omicron_common::api::external::http_pagination::id_pagination; use omicron_common::api::external::http_pagination::marker_for_id; use omicron_common::api::external::http_pagination::marker_for_name; use omicron_common::api::external::http_pagination::marker_for_name_or_id; @@ -2577,7 +2576,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query>, path_params: Path, ) -> Result>, HttpError> { @@ -2589,8 +2588,8 @@ impl NexusExternalApi for NexusExternalApiImpl { let path = path_params.into_inner(); let query = query_params.into_inner(); let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanById::from_query(&query)?; - let paginated_by = id_pagination(&pag_params, scan_params)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; let group_selector = params::AffinityGroupSelector { project: scan_params.selector.project.clone(), @@ -2605,10 +2604,10 @@ impl NexusExternalApi for NexusExternalApiImpl { &paginated_by, ) .await?; - Ok(HttpResponseOk(ScanById::results_page( + Ok(HttpResponseOk(ScanByNameOrId::results_page( &query, affinity_group_member_instances, - &marker_for_id, + &marker_for_name_or_id, )?)) }; apictx @@ -2912,7 +2911,7 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query>, path_params: Path, ) -> Result>, HttpError> { @@ -2924,7 +2923,8 @@ impl NexusExternalApi for NexusExternalApiImpl { let path = path_params.into_inner(); let query = query_params.into_inner(); let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanById::from_query(&query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; let group_selector = params::AntiAffinityGroupSelector { project: scan_params.selector.project.clone(), @@ -2936,13 +2936,13 @@ impl NexusExternalApi for NexusExternalApiImpl { .anti_affinity_group_member_list( &opctx, &group_lookup, - &pag_params, + &paginated_by, ) .await?; - Ok(HttpResponseOk(ScanById::results_page( + Ok(HttpResponseOk(ScanByNameOrId::results_page( &query, group_members, - &marker_for_id, + &marker_for_name_or_id, )?)) }; apictx From de905ff55f8238821b8715f9cd1f4cfe7a02f085 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 28 Feb 2025 14:58:24 -0800 Subject: [PATCH 77/84] Fix pagination, add tests --- nexus/db-queries/src/db/datastore/affinity.rs | 268 ++++++++++++++++-- nexus/external-api/src/lib.rs | 8 +- nexus/src/app/affinity.rs | 3 +- nexus/src/external_api/http_entrypoints.rs | 8 +- openapi/nexus.json | 4 +- 5 files changed, 259 insertions(+), 32 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 3c18a0db77f..70aec1c2fb3 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -379,7 +379,8 @@ impl DataStore { opctx.authorize(authz::Action::Read, authz_affinity_group).await?; let mut query = QueryBuilder::new() - .sql(" + .sql( + " SELECT instance_id as id, name FROM affinity_group_instance_membership INNER JOIN instance @@ -403,26 +404,27 @@ impl DataStore { PaginatedBy::Id(DataPageParams { marker, .. }) => { if let Some(id) = marker { query = query - .sql("WHERE id ") + .sql("AND id ") .sql(if asc { ">" } else { "<" }) .sql(" ") .param() .bind::(**id); }; query = query.sql(" ORDER BY id "); - }, + } PaginatedBy::Name(DataPageParams { marker, .. }) => { if let Some(name) = marker { query = query - .sql("WHERE name ") + .sql("AND name ") .sql(if asc { ">" } else { "<" }) .sql(" ") .param() - .bind::(Name((*name).clone())); + .bind::(Name( + (*name).clone(), + )); }; query = query.sql(" ORDER BY name "); - - }, + } } if asc { query = query.sql("ASC "); @@ -430,16 +432,13 @@ impl DataStore { query = query.sql("DESC "); } - query = - query.sql(" LIMIT ").param().bind::( - i64::from(limit.get()), - ); + query = query + .sql(" LIMIT ") + .param() + .bind::(i64::from(limit.get())); query - .query::<( - diesel::sql_types::Uuid, - diesel::sql_types::Text, - )>() + .query::<(diesel::sql_types::Uuid, diesel::sql_types::Text)>() .load_async::<(Uuid, Name)>( &*self.pool_connection_authorized(opctx).await?, ) @@ -510,7 +509,7 @@ impl DataStore { .bind::(**id); }; query = query.sql(" ORDER BY id "); - }, + } PaginatedBy::Name(DataPageParams { marker, .. }) => { if let Some(name) = marker { query = query @@ -518,11 +517,12 @@ impl DataStore { .sql(if asc { ">" } else { "<" }) .sql(" ") .param() - .bind::(Name((*name).clone())); + .bind::(Name( + (*name).clone(), + )); }; query = query.sql(" ORDER BY name "); - - }, + } } if asc { query = query.sql("ASC "); @@ -530,10 +530,10 @@ impl DataStore { query = query.sql("DESC "); } - query = - query.sql(" LIMIT ").param().bind::( - i64::from(limit.get()), - ); + query = query + .sql(" LIMIT ") + .param() + .bind::(i64::from(limit.get())); query .query::<( @@ -1952,6 +1952,174 @@ mod tests { logctx.cleanup_successful(); } + #[tokio::test] + async fn affinity_group_membership_list_extended() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_list_extended"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams = PaginatedBy::Id(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert!(members.is_empty()); + + // Add some instances, so we have data to list over. + + const INSTANCE_COUNT: usize = 6; + + let mut members = Vec::new(); + for i in 0..INSTANCE_COUNT { + let name = format!("instance-{i}"); + let instance = create_stopped_instance_record( + &opctx, + &datastore, + &authz_project, + &name, + ) + .await; + + // Add the instance as a member to the group + let member = external::AffinityGroupMember::Instance { + id: instance, + name: name.try_into().unwrap(), + }; + datastore + .affinity_group_member_instance_add( + &opctx, + &authz_group, + instance, + ) + .await + .unwrap(); + members.push(member); + } + + // Order by UUID + members.sort_unstable_by_key(|m1| m1.id()); + + // We can list all members + let pagparams = PaginatedBy::Id(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members); + + // We can paginate over the results + let marker = members[2].id(); + let pagparams = PaginatedBy::Id(DataPageParams { + marker: Some(&marker), + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members[3..]); + + // We can list limited results + let pagparams = PaginatedBy::Id(DataPageParams { + marker: Some(&marker), + limit: NonZeroU32::new(2).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members[3..5]); + + // We can list in descending order too + members.reverse(); + let pagparams = PaginatedBy::Id(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Descending, + }); + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members); + + // Order by name + members.sort_unstable_by_key(|m1| m1.name().clone()); + + // We can list all members + let pagparams = PaginatedBy::Name(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members); + let marker = members[2].name(); + let pagparams = PaginatedBy::Name(DataPageParams { + marker: Some(marker), + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members[3..]); + + // We can list in descending order too + members.reverse(); + let pagparams = PaginatedBy::Name(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Descending, + }); + let observed_members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagparams) + .await + .unwrap(); + assert_eq!(observed_members, members); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + // Anti-affinity group member listing has a slightly more complicated // implementation, because it queries multiple tables and UNIONs them // together. @@ -2119,7 +2287,59 @@ mod tests { let pagparams = PaginatedBy::Id(DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), - direction: dropshot::PaginationOrder::Descending + direction: dropshot::PaginationOrder::Descending, + }); + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members); + + // Order by name, regardless of member type + members.sort_unstable_by_key(|m1| m1.name().clone()); + + // We can list all members + let pagparams = PaginatedBy::Name(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members); + let marker = members[2].name(); + let pagparams = PaginatedBy::Name(DataPageParams { + marker: Some(marker), + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }); + + let observed_members = datastore + .anti_affinity_group_member_list( + &opctx, + &authz_aa_group, + &pagparams, + ) + .await + .unwrap(); + assert_eq!(observed_members, members[3..]); + + // We can list in descending order too + members.reverse(); + let pagparams = PaginatedBy::Name(DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Descending, }); let observed_members = datastore .anti_affinity_group_member_list( diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 55678a1b22c..3c64c57434c 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -1297,7 +1297,9 @@ pub trait NexusExternalApi { }] async fn affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query< + PaginatedByNameOrId, + >, path_params: Path, ) -> Result>, HttpError>; @@ -1405,7 +1407,9 @@ pub trait NexusExternalApi { }] async fn anti_affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query< + PaginatedByNameOrId, + >, path_params: Path, ) -> Result>, HttpError>; diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 60384a5d804..d1d11aa960f 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -245,8 +245,7 @@ impl super::Nexus { Ok(self .db_datastore .affinity_group_member_list(opctx, &authz_affinity_group, pagparams) - .await? - ) + .await?) } pub(crate) async fn anti_affinity_group_member_list( diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index ef7dab2d739..94ca2ae078f 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -2576,7 +2576,9 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query< + PaginatedByNameOrId, + >, path_params: Path, ) -> Result>, HttpError> { @@ -2911,7 +2913,9 @@ impl NexusExternalApi for NexusExternalApiImpl { async fn anti_affinity_group_member_list( rqctx: RequestContext, - query_params: Query>, + query_params: Query< + PaginatedByNameOrId, + >, path_params: Path, ) -> Result>, HttpError> { diff --git a/openapi/nexus.json b/openapi/nexus.json index 41ebe8a3806..7ef45868b26 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -975,7 +975,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrIdSortMode" } }, { @@ -1451,7 +1451,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrIdSortMode" } }, { From 20e6bbb2319d9799f723519669530bb08278d19f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 28 Feb 2025 15:00:18 -0800 Subject: [PATCH 78/84] Clippy --- nexus/src/app/affinity.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index d1d11aa960f..c3d414c3f62 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -242,10 +242,9 @@ impl super::Nexus { let (.., authz_affinity_group) = affinity_group_lookup .lookup_for(authz::Action::ListChildren) .await?; - Ok(self - .db_datastore + self.db_datastore .affinity_group_member_list(opctx, &authz_affinity_group, pagparams) - .await?) + .await } pub(crate) async fn anti_affinity_group_member_list( From b5138eb2b3a2578d5fca77224577e8a3cfabb3b2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 28 Feb 2025 15:59:29 -0800 Subject: [PATCH 79/84] Plumb state out too --- common/src/api/external/mod.rs | 5 +- nexus/db-model/src/affinity.rs | 4 + nexus/db-queries/src/db/datastore/affinity.rs | 168 +++++++++++++++--- nexus/db-queries/src/db/datastore/instance.rs | 23 ++- nexus/src/app/affinity.rs | 8 + openapi/nexus.json | 12 +- 6 files changed, 181 insertions(+), 39 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 17f49f47524..6b4e39efb0d 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1067,6 +1067,7 @@ pub struct IdentityMetadataUpdateParams { Debug, Deserialize, Eq, + Hash, Ord, PartialEq, PartialOrd, @@ -1322,7 +1323,7 @@ pub enum FailureDomain { #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AffinityGroupMember { /// An instance belonging to this group - Instance { id: InstanceUuid, name: Name }, + Instance { id: InstanceUuid, name: Name, run_state: InstanceState }, } impl SimpleIdentityOrName for AffinityGroupMember { @@ -1352,7 +1353,7 @@ pub enum AntiAffinityGroupMember { AffinityGroup { id: AffinityGroupUuid, name: Name }, /// An instance belonging to this group - Instance { id: InstanceUuid, name: Name }, + Instance { id: InstanceUuid, name: Name, run_state: InstanceState }, } impl SimpleIdentityOrName for AntiAffinityGroupMember { diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index 01063f249e9..22ec64670de 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -231,10 +231,12 @@ impl AffinityGroupInstanceMembership { pub fn to_external( self, member_name: external::Name, + run_state: external::InstanceState, ) -> external::AffinityGroupMember { external::AffinityGroupMember::Instance { id: self.instance_id.into(), name: member_name, + run_state, } } } @@ -257,10 +259,12 @@ impl AntiAffinityGroupInstanceMembership { pub fn to_external( self, member_name: external::Name, + run_state: external::InstanceState, ) -> external::AntiAffinityGroupMember { external::AntiAffinityGroupMember::Instance { id: self.instance_id.into(), name: member_name, + run_state, } } } diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 70aec1c2fb3..5a89f9b9fde 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -10,6 +10,7 @@ use crate::authz::ApiResource; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; +use crate::db::datastore::InstanceAndActiveVmm; use crate::db::datastore::OpContext; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; @@ -21,8 +22,12 @@ use crate::db::model::AntiAffinityGroup; use crate::db::model::AntiAffinityGroupAffinityMembership; use crate::db::model::AntiAffinityGroupInstanceMembership; use crate::db::model::AntiAffinityGroupUpdate; +use crate::db::model::InstanceState; +use crate::db::model::InstanceStateEnum; use crate::db::model::Name; use crate::db::model::Project; +use crate::db::model::VmmState; +use crate::db::model::VmmStateEnum; use crate::db::pagination::paginated; use crate::db::raw_query_builder::QueryBuilder; use crate::transaction_retry::OptionalError; @@ -381,15 +386,26 @@ impl DataStore { let mut query = QueryBuilder::new() .sql( " - SELECT instance_id as id, name + SELECT * FROM ( + SELECT + instance.id as id, + instance.name as name, + instance.state, + instance.migration_id, + vmm.state FROM affinity_group_instance_membership INNER JOIN instance ON instance.id = affinity_group_instance_membership.instance_id - WHERE instance.time_deleted IS NULL AND group_id = ", + LEFT JOIN vmm + ON instance.active_propolis_id = vmm.id + WHERE + instance.time_deleted IS NULL AND + vmm.time_deleted IS NULL AND + group_id = ", ) .param() .bind::(authz_affinity_group.id()) - .sql(" "); + .sql(") "); let (direction, limit) = match pagparams { PaginatedBy::Id(p) => (p.direction, p.limit), @@ -404,7 +420,7 @@ impl DataStore { PaginatedBy::Id(DataPageParams { marker, .. }) => { if let Some(id) = marker { query = query - .sql("AND id ") + .sql("WHERE id ") .sql(if asc { ">" } else { "<" }) .sql(" ") .param() @@ -415,7 +431,7 @@ impl DataStore { PaginatedBy::Name(DataPageParams { marker, .. }) => { if let Some(name) = marker { query = query - .sql("AND name ") + .sql("WHERE name ") .sql(if asc { ">" } else { "<" }) .sql(" ") .param() @@ -438,17 +454,28 @@ impl DataStore { .bind::(i64::from(limit.get())); query - .query::<(diesel::sql_types::Uuid, diesel::sql_types::Text)>() - .load_async::<(Uuid, Name)>( + .query::<( + diesel::sql_types::Uuid, + diesel::sql_types::Text, + InstanceStateEnum, + diesel::sql_types::Nullable, + diesel::sql_types::Nullable, + )>() + .load_async::<(Uuid, Name, InstanceState, Option, Option)>( &*self.pool_connection_authorized(opctx).await?, ) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .into_iter() - .map(|(id, name)| { + .map(|(id, name, instance_state, migration_id, vmm_state)| { Ok(external::AffinityGroupMember::Instance { id: InstanceUuid::from_untyped_uuid(id), name: name.into(), + run_state: InstanceAndActiveVmm::determine_effective_state_inner( + instance_state, + migration_id, + vmm_state, + ), }) }) .collect() @@ -473,26 +500,43 @@ impl DataStore { let mut query = QueryBuilder::new() .sql( - "SELECT id,name,label + "SELECT id,name,label,instance_state,migration_id,vmm_state FROM ( - SELECT instance_id as id, name, 'instance' as label + SELECT + instance.id as id, + instance.name as name, + 'instance' as label, + instance.state as instance_state, + instance.migration_id as migration_id, + vmm.state as vmm_state FROM anti_affinity_group_instance_membership INNER JOIN instance ON instance.id = anti_affinity_group_instance_membership.instance_id - WHERE instance.time_deleted IS NULL AND - group_id = ", + LEFT JOIN vmm + ON instance.active_propolis_id = vmm.id + WHERE + instance.time_deleted IS NULL AND + vmm.time_deleted IS NULL AND + group_id = ", ) .param() .bind::(authz_anti_affinity_group.id()) .sql( " UNION - SELECT affinity_group_id as id, name, 'affinity_group' as label + SELECT + affinity_group.id as id, + affinity_group.name as name, + 'affinity_group' as label, + NULL as instance_state, + NULL as migration_id, + NULL as vmm_state FROM anti_affinity_group_affinity_membership INNER JOIN affinity_group ON affinity_group.id = anti_affinity_group_affinity_membership.affinity_group_id - WHERE affinity_group.time_deleted IS NULL AND - anti_affinity_group_id = ", + WHERE + affinity_group.time_deleted IS NULL AND + anti_affinity_group_id = ", ) .param() .bind::(authz_anti_affinity_group.id()) @@ -540,24 +584,46 @@ impl DataStore { diesel::sql_types::Uuid, diesel::sql_types::Text, diesel::sql_types::Text, + diesel::sql_types::Nullable, + diesel::sql_types::Nullable, + diesel::sql_types::Nullable, )>() - .load_async::<(Uuid, Name, String)>( + .load_async::<( + Uuid, + Name, + String, + Option, + Option, + Option, + )>( &*self.pool_connection_authorized(opctx).await?, ) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .into_iter() - .map(|(id, name, label)| { + .map(|(id, name, label, instance_state, migration_id, vmm_state)| { use external::AntiAffinityGroupMember as Member; match label.as_str() { "affinity_group" => Ok(Member::AffinityGroup { id: AffinityGroupUuid::from_untyped_uuid(id), name: name.into(), }), - "instance" => Ok(Member::Instance { - id: InstanceUuid::from_untyped_uuid(id), - name: name.into(), - }), + "instance" => { + let Some(instance_state) = instance_state else { + return Err(external::Error::internal_error( + "Anti-Affinity instance member missing state in database" + )); + }; + Ok(Member::Instance { + id: InstanceUuid::from_untyped_uuid(id), + name: name.into(), + run_state: InstanceAndActiveVmm::determine_effective_state_inner( + instance_state, + migration_id, + vmm_state, + ), + }) + }, other => Err(external::Error::internal_error(&format!( "Unexpected label from database query: {other}" ))), @@ -577,6 +643,7 @@ impl DataStore { use db::schema::affinity_group_instance_membership::dsl; use db::schema::instance::dsl as instance_dsl; + use db::schema::vmm::dsl as vmm_dsl; dsl::affinity_group_instance_membership .filter(dsl::group_id.eq(authz_affinity_group.id())) .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) @@ -585,13 +652,34 @@ impl DataStore { .on(instance_dsl::id.eq(dsl::instance_id)), ) .filter(instance_dsl::time_deleted.is_null()) + .left_join(vmm_dsl::vmm.on( + instance_dsl::active_propolis_id.eq(vmm_dsl::id.nullable()), + )) + .filter(vmm_dsl::time_deleted.is_null()) .select(( AffinityGroupInstanceMembership::as_select(), instance_dsl::name, + instance_dsl::state, + instance_dsl::migration_id, + vmm_dsl::state.nullable(), )) - .get_result_async::<(AffinityGroupInstanceMembership, Name)>(&*conn) - .await - .map(|(member, name)| member.to_external(name.into())) + .get_result_async::<( + AffinityGroupInstanceMembership, + Name, + InstanceState, + Option, + Option, + )>(&*conn) + .await + .map(|(member, name, instance_state, migration_id, vmm_state)| { + let run_state = + InstanceAndActiveVmm::determine_effective_state_inner( + instance_state, + migration_id, + vmm_state, + ); + member.to_external(name.into(), run_state) + }) .map_err(|e| { public_error_from_diesel( e, @@ -614,6 +702,7 @@ impl DataStore { use db::schema::anti_affinity_group_instance_membership::dsl; use db::schema::instance::dsl as instance_dsl; + use db::schema::vmm::dsl as vmm_dsl; dsl::anti_affinity_group_instance_membership .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) @@ -622,15 +711,34 @@ impl DataStore { .on(instance_dsl::id.eq(dsl::instance_id)), ) .filter(instance_dsl::time_deleted.is_null()) + .left_join(vmm_dsl::vmm.on( + instance_dsl::active_propolis_id.eq(vmm_dsl::id.nullable()), + )) + .filter(vmm_dsl::time_deleted.is_null()) .select(( AntiAffinityGroupInstanceMembership::as_select(), instance_dsl::name, + instance_dsl::state, + instance_dsl::migration_id, + vmm_dsl::state.nullable(), )) - .get_result_async::<(AntiAffinityGroupInstanceMembership, Name)>( - &*conn, - ) - .await - .map(|(member, name)| member.to_external(name.into())) + .get_result_async::<( + AntiAffinityGroupInstanceMembership, + Name, + InstanceState, + Option, + Option, + )>(&*conn) + .await + .map(|(member, name, instance_state, migration_id, vmm_state)| { + let run_state = + InstanceAndActiveVmm::determine_effective_state_inner( + instance_state, + migration_id, + vmm_state, + ); + member.to_external(name.into(), run_state) + }) .map_err(|e| { public_error_from_diesel( e, @@ -2009,6 +2117,7 @@ mod tests { let member = external::AffinityGroupMember::Instance { id: instance, name: name.try_into().unwrap(), + run_state: external::InstanceState::Stopped, }; datastore .affinity_group_member_instance_add( @@ -2188,6 +2297,7 @@ mod tests { let member = external::AntiAffinityGroupMember::Instance { id: instance, name: name.try_into().unwrap(), + run_state: external::InstanceState::Stopped, }; datastore .anti_affinity_group_member_instance_add( diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index fc3fcf25cb9..ddcca8df255 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -114,12 +114,25 @@ impl InstanceAndActiveVmm { instance: &Instance, active_vmm: Option<&Vmm>, ) -> external::InstanceState { - use crate::db::model::InstanceState; - use crate::db::model::VmmState; - let instance_state = instance.runtime_state.nexus_state; + let migration_id = instance.runtime_state.migration_id; let vmm_state = active_vmm.map(|vmm| vmm.runtime.state); + Self::determine_effective_state_inner( + instance_state, + migration_id, + vmm_state, + ) + } + + pub fn determine_effective_state_inner( + instance_state: InstanceState, + migration_id: Option, + vmm_state: Option, + ) -> external::InstanceState { + use crate::db::model::InstanceState; + use crate::db::model::VmmState; + // We want to only report that an instance is `Stopped` when a new // `instance-start` saga is able to proceed. That means that: match (instance_state, vmm_state) { @@ -145,9 +158,7 @@ impl InstanceAndActiveVmm { // instance-update saga will come along and remove the active VMM // and migration IDs. // - (InstanceState::Vmm, Some(_)) - if instance.runtime_state.migration_id.is_some() => - { + (InstanceState::Vmm, Some(_)) if migration_id.is_some() => { external::InstanceState::Migrating } // - An instance with a "stopped" or "destroyed" VMM needs to be diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index c3d414c3f62..4936e34b59d 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -351,6 +351,10 @@ impl super::Nexus { Ok(external::AffinityGroupMember::Instance { id: member, name: instance.name().clone(), + // TODO: This is kinda a lie - the current implementation of + // "affinity_group_member_instance_add" relies on the instance + // note having a VMM, but that might change in the future. + run_state: external::InstanceState::Stopped, }) } @@ -377,6 +381,10 @@ impl super::Nexus { Ok(external::AntiAffinityGroupMember::Instance { id: member, name: instance.name().clone(), + // TODO: This is kinda a lie - the current implementation of + // "anti_affinity_group_member_instance_add" relies on the instance + // note having a VMM, but that might change in the future. + run_state: external::InstanceState::Stopped, }) } diff --git a/openapi/nexus.json b/openapi/nexus.json index 7ef45868b26..bfe4dd6f66a 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12373,11 +12373,15 @@ }, "name": { "$ref": "#/components/schemas/Name" + }, + "run_state": { + "$ref": "#/components/schemas/InstanceState" } }, "required": [ "id", - "name" + "name", + "run_state" ] } }, @@ -12695,11 +12699,15 @@ }, "name": { "$ref": "#/components/schemas/Name" + }, + "run_state": { + "$ref": "#/components/schemas/InstanceState" } }, "required": [ "id", - "name" + "name", + "run_state" ] } }, From 6e369d95f4db72e98371aa166899eaa34df5f72f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 17 Mar 2025 17:22:17 -0700 Subject: [PATCH 80/84] Undo group-of-groups --- common/src/api/external/mod.rs | 8 - nexus/db-model/src/affinity.rs | 29 - nexus/db-model/src/schema.rs | 8 - nexus/db-queries/src/db/datastore/affinity.rs | 794 +----------------- nexus/db-queries/src/db/datastore/sled.rs | 388 --------- .../src/db/queries/sled_reservation.rs | 144 +--- nexus/external-api/output/nexus_tags.txt | 3 - nexus/external-api/src/lib.rs | 36 - nexus/src/app/affinity.rs | 73 -- nexus/src/external_api/http_entrypoints.rs | 136 --- nexus/tests/integration_tests/endpoints.rs | 21 - nexus/tests/integration_tests/unauthorized.rs | 6 - nexus/types/src/external_api/params.rs | 6 - openapi/nexus.json | 183 ---- schema/crdb/dbinit.sql | 19 - 15 files changed, 77 insertions(+), 1777 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 2b818367890..3d1b4330ba1 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -22,7 +22,6 @@ use dropshot::HttpError; pub use dropshot::PaginationOrder; pub use error::*; use futures::stream::BoxStream; -use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use oxnet::IpNet; @@ -1349,9 +1348,6 @@ impl SimpleIdentityOrName for AffinityGroupMember { )] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { - /// An affinity group belonging to this group - AffinityGroup { id: AffinityGroupUuid, name: Name }, - /// An instance belonging to this group Instance { id: InstanceUuid, name: Name, run_state: InstanceState }, } @@ -1359,9 +1355,6 @@ pub enum AntiAffinityGroupMember { impl SimpleIdentityOrName for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { - AntiAffinityGroupMember::AffinityGroup { id, .. } => { - *id.as_untyped_uuid() - } AntiAffinityGroupMember::Instance { id, .. } => { *id.as_untyped_uuid() } @@ -1370,7 +1363,6 @@ impl SimpleIdentityOrName for AntiAffinityGroupMember { fn name(&self) -> &Name { match self { - AntiAffinityGroupMember::AffinityGroup { name, .. } => name, AntiAffinityGroupMember::Instance { name, .. } => name, } } diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index 197780ef2db..55a085e905d 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -11,7 +11,6 @@ use super::impl_enum_type; use crate::schema::affinity_group; use crate::schema::affinity_group_instance_membership; use crate::schema::anti_affinity_group; -use crate::schema::anti_affinity_group_affinity_membership; use crate::schema::anti_affinity_group_instance_membership; use crate::typed_uuid::DbTypedUuid; use chrono::{DateTime, Utc}; @@ -270,31 +269,3 @@ impl AntiAffinityGroupInstanceMembership { } } } - -#[derive(Queryable, Insertable, Clone, Debug, Selectable)] -#[diesel(table_name = anti_affinity_group_affinity_membership)] -pub struct AntiAffinityGroupAffinityMembership { - pub anti_affinity_group_id: DbTypedUuid, - pub affinity_group_id: DbTypedUuid, -} - -impl AntiAffinityGroupAffinityMembership { - pub fn new( - anti_affinity_group_id: AntiAffinityGroupUuid, - affinity_group_id: AffinityGroupUuid, - ) -> Self { - Self { - anti_affinity_group_id: anti_affinity_group_id.into(), - affinity_group_id: affinity_group_id.into(), - } - } - pub fn to_external( - self, - member_name: external::Name, - ) -> external::AntiAffinityGroupMember { - external::AntiAffinityGroupMember::AffinityGroup { - id: self.affinity_group_id.into(), - name: member_name, - } - } -} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 61c67023925..a45c8b6d20b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -510,13 +510,6 @@ table! { } } -table! { - anti_affinity_group_affinity_membership (anti_affinity_group_id, affinity_group_id) { - anti_affinity_group_id -> Uuid, - affinity_group_id -> Uuid, - } -} - table! { metric_producer (id) { id -> Uuid, @@ -2064,7 +2057,6 @@ allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( anti_affinity_group, - anti_affinity_group_affinity_membership, anti_affinity_group_instance_membership, affinity_group, affinity_group_instance_membership, diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 2cf8b081051..c0b53a924c8 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -19,7 +19,6 @@ use crate::db::model::AffinityGroup; use crate::db::model::AffinityGroupInstanceMembership; use crate::db::model::AffinityGroupUpdate; use crate::db::model::AntiAffinityGroup; -use crate::db::model::AntiAffinityGroupAffinityMembership; use crate::db::model::AntiAffinityGroupInstanceMembership; use crate::db::model::AntiAffinityGroupUpdate; use crate::db::model::InstanceState; @@ -246,20 +245,6 @@ impl DataStore { })?; } - // If this affinity group is a member in other anti-affinity - // groups, remove those memberships - { - use db::schema::anti_affinity_group_affinity_membership::dsl as member_dsl; - diesel::delete(member_dsl::anti_affinity_group_affinity_membership) - .filter(member_dsl::affinity_group_id.eq(authz_affinity_group.id())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; - } Ok(()) } }) @@ -349,18 +334,6 @@ impl DataStore { }) })?; } - { - use db::schema::anti_affinity_group_affinity_membership::dsl as member_dsl; - diesel::delete(member_dsl::anti_affinity_group_affinity_membership) - .filter(member_dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; - } Ok(()) } @@ -500,12 +473,11 @@ impl DataStore { let mut query = QueryBuilder::new() .sql( - "SELECT id,name,label,instance_state,migration_id,vmm_state + "SELECT id,name,instance_state,migration_id,vmm_state FROM ( SELECT instance.id as id, instance.name as name, - 'instance' as label, instance.state as instance_state, instance.migration_id as migration_id, vmm.state as vmm_state @@ -521,25 +493,6 @@ impl DataStore { ) .param() .bind::(authz_anti_affinity_group.id()) - .sql( - " - UNION - SELECT - affinity_group.id as id, - affinity_group.name as name, - 'affinity_group' as label, - NULL as instance_state, - NULL as migration_id, - NULL as vmm_state - FROM anti_affinity_group_affinity_membership - INNER JOIN affinity_group - ON affinity_group.id = anti_affinity_group_affinity_membership.affinity_group_id - WHERE - affinity_group.time_deleted IS NULL AND - anti_affinity_group_id = ", - ) - .param() - .bind::(authz_anti_affinity_group.id()) .sql(") "); match pagparams { @@ -583,7 +536,6 @@ impl DataStore { .query::<( diesel::sql_types::Uuid, diesel::sql_types::Text, - diesel::sql_types::Text, diesel::sql_types::Nullable, diesel::sql_types::Nullable, diesel::sql_types::Nullable, @@ -591,7 +543,6 @@ impl DataStore { .load_async::<( Uuid, Name, - String, Option, Option, Option, @@ -601,33 +552,21 @@ impl DataStore { .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .into_iter() - .map(|(id, name, label, instance_state, migration_id, vmm_state)| { - use external::AntiAffinityGroupMember as Member; - match label.as_str() { - "affinity_group" => Ok(Member::AffinityGroup { - id: AffinityGroupUuid::from_untyped_uuid(id), - name: name.into(), - }), - "instance" => { - let Some(instance_state) = instance_state else { - return Err(external::Error::internal_error( - "Anti-Affinity instance member missing state in database" - )); - }; - Ok(Member::Instance { - id: InstanceUuid::from_untyped_uuid(id), - name: name.into(), - run_state: InstanceAndActiveVmm::determine_effective_state_inner( - instance_state, - migration_id, - vmm_state, - ), - }) - }, - other => Err(external::Error::internal_error(&format!( - "Unexpected label from database query: {other}" - ))), - } + .map(|(id, name, instance_state, migration_id, vmm_state)| { + let Some(instance_state) = instance_state else { + return Err(external::Error::internal_error( + "Anti-Affinity instance member missing state in database" + )); + }; + Ok(external::AntiAffinityGroupMember::Instance { + id: InstanceUuid::from_untyped_uuid(id), + name: name.into(), + run_state: InstanceAndActiveVmm::determine_effective_state_inner( + instance_state, + migration_id, + vmm_state, + ), + }) }) .collect() } @@ -750,51 +689,6 @@ impl DataStore { }) } - pub async fn anti_affinity_group_member_affinity_group_view( - &self, - opctx: &OpContext, - authz_anti_affinity_group: &authz::AntiAffinityGroup, - affinity_group_id: AffinityGroupUuid, - ) -> Result { - opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; - let conn = self.pool_connection_authorized(opctx).await?; - - use db::schema::affinity_group::dsl as affinity_group_dsl; - use db::schema::anti_affinity_group_affinity_membership::dsl; - dsl::anti_affinity_group_affinity_membership - .filter( - dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id()), - ) - .filter( - dsl::affinity_group_id - .eq(affinity_group_id.into_untyped_uuid()), - ) - .inner_join( - affinity_group_dsl::affinity_group - .on(affinity_group_dsl::id.eq(dsl::affinity_group_id)), - ) - .select(( - AntiAffinityGroupAffinityMembership::as_select(), - affinity_group_dsl::name, - )) - .get_result_async::<(AntiAffinityGroupAffinityMembership, Name)>( - &*conn, - ) - .await - .map(|(member, name)| member.to_external(name.into())) - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::AntiAffinityGroupMember, - LookupType::by_id( - affinity_group_id.into_untyped_uuid(), - ), - ), - ) - }) - } - pub async fn affinity_group_member_instance_add( &self, opctx: &OpContext, @@ -1034,133 +928,6 @@ impl DataStore { Ok(()) } - pub async fn anti_affinity_group_member_affinity_group_add( - &self, - opctx: &OpContext, - authz_anti_affinity_group: &authz::AntiAffinityGroup, - affinity_group_id: AffinityGroupUuid, - ) -> Result<(), Error> { - opctx - .authorize(authz::Action::Modify, authz_anti_affinity_group) - .await?; - - let err = OptionalError::new(); - let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_affinity_group_add") - .transaction(&conn, |conn| { - let err = err.clone(); - use db::schema::anti_affinity_group::dsl as anti_affinity_group_dsl; - use db::schema::affinity_group::dsl as affinity_group_dsl; - use db::schema::affinity_group_instance_membership::dsl as a_instance_membership_dsl; - use db::schema::anti_affinity_group_affinity_membership::dsl as aa_affinity_membership_dsl; - use db::schema::sled_resource_vmm::dsl as resource_dsl; - - async move { - // Check that the anti-affinity group exists - anti_affinity_group_dsl::anti_affinity_group - .filter(anti_affinity_group_dsl::time_deleted.is_null()) - .filter(anti_affinity_group_dsl::id.eq(authz_anti_affinity_group.id())) - .select(anti_affinity_group_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource( - authz_anti_affinity_group, - ), - ) - }) - })?; - - // Check that the affinity group exists - affinity_group_dsl::affinity_group - .filter(affinity_group_dsl::time_deleted.is_null()) - .filter(affinity_group_dsl::id.eq(affinity_group_id.into_untyped_uuid())) - .select(affinity_group_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::AffinityGroup, - LookupType::ById(affinity_group_id.into_untyped_uuid()) - ), - ) - }) - })?; - - - // Check that the affinity group's members are not reserved. - let has_reservation: bool = diesel::select( - diesel::dsl::exists( - a_instance_membership_dsl::affinity_group_instance_membership - .inner_join( - resource_dsl::sled_resource_vmm - .on(resource_dsl::instance_id.eq(a_instance_membership_dsl::instance_id.nullable())) - ) - .filter(a_instance_membership_dsl::group_id.eq( - affinity_group_id.into_untyped_uuid() - )) - ) - ).get_result_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::Server, - ) - }) - })?; - - // NOTE: This check prevents us from violating affinity rules with "policy = - // fail" stances, but it is possible that running instances already would - // satisfy the affinity rules proposed by this new group addition. - // - // It would be possible to remove this error if we replaced it with affinity - // rule checks. - if has_reservation { - return Err(err.bail(Error::invalid_request( - "Affinity group with running instances cannot be \ - added to an anti-affinity group. Try stopping them first.".to_string() - ))); - } - - diesel::insert_into(aa_affinity_membership_dsl::anti_affinity_group_affinity_membership) - .values(AntiAffinityGroupAffinityMembership::new( - AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), - affinity_group_id, - )) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::AntiAffinityGroupMember, - &affinity_group_id.to_string(), - ), - ) - }) - })?; - Ok(()) - } - }) - .await - .map_err(|e| { - if let Some(err) = err.take() { - return err; - } - public_error_from_diesel(e, ErrorHandler::Server) - })?; - Ok(()) - } - pub async fn instance_affinity_group_memberships_delete( &self, opctx: &OpContext, @@ -1319,72 +1086,6 @@ impl DataStore { })?; Ok(()) } - - /// Deletes an anti-affinity member, when that member is an affinity group - pub async fn anti_affinity_group_member_affinity_group_delete( - &self, - opctx: &OpContext, - authz_anti_affinity_group: &authz::AntiAffinityGroup, - affinity_group_id: AffinityGroupUuid, - ) -> Result<(), Error> { - opctx - .authorize(authz::Action::Modify, authz_anti_affinity_group) - .await?; - - let err = OptionalError::new(); - let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_affinity_group_delete") - .transaction(&conn, |conn| { - let err = err.clone(); - use db::schema::anti_affinity_group::dsl as group_dsl; - use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; - - async move { - // Check that the anti-affinity group exists - group_dsl::anti_affinity_group - .filter(group_dsl::time_deleted.is_null()) - .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) - .select(group_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource( - authz_anti_affinity_group, - ), - ) - }) - })?; - - let rows = diesel::delete(membership_dsl::anti_affinity_group_affinity_membership) - .filter(membership_dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id())) - .filter(membership_dsl::affinity_group_id.eq(affinity_group_id.into_untyped_uuid())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; - if rows == 0 { - return Err(err.bail(LookupType::ById(affinity_group_id.into_untyped_uuid()).into_not_found( - ResourceType::AntiAffinityGroupMember, - ))); - } - Ok(()) - } - }) - .await - .map_err(|e| { - if let Some(err) = err.take() { - return err; - } - public_error_from_diesel(e, ErrorHandler::Server) - })?; - Ok(()) - } } #[cfg(test)] @@ -2276,10 +1977,9 @@ mod tests { .unwrap(); assert!(members.is_empty()); - // Add some groups and instances, so we have data to list over. + // Add some instances, so we have data to list over. - const INSTANCE_COUNT: usize = 3; - const AFFINITY_GROUP_COUNT: usize = 3; + const INSTANCE_COUNT: usize = 6; let mut members = Vec::new(); @@ -2310,36 +2010,7 @@ mod tests { members.push(member); } - for i in 0..AFFINITY_GROUP_COUNT { - let name = format!("affinity-{i}"); - let affinity_group = create_affinity_group( - &opctx, - &datastore, - &authz_project, - &name, - ) - .await - .unwrap(); - - let affinity_group_id = - AffinityGroupUuid::from_untyped_uuid(affinity_group.id()); - // Add the instance as a member to the group - let member = external::AntiAffinityGroupMember::AffinityGroup { - id: affinity_group_id, - name: name.try_into().unwrap(), - }; - datastore - .anti_affinity_group_member_affinity_group_add( - &opctx, - &authz_aa_group, - affinity_group_id, - ) - .await - .unwrap(); - members.push(member); - } - - // Order by UUID, regardless of member type + // Order by UUID members.sort_unstable_by_key(|m1| m1.id()); // We can list all members @@ -2409,7 +2080,7 @@ mod tests { .unwrap(); assert_eq!(observed_members, members); - // Order by name, regardless of member type + // Order by name members.sort_unstable_by_key(|m1| m1.name().clone()); // We can list all members @@ -2689,166 +2360,6 @@ mod tests { logctx.cleanup_successful(); } - #[tokio::test] - async fn anti_affinity_group_membership_add_remove_group_with_vmm() { - // Setup - let logctx = dev::test_setup_log( - "anti_affinity_group_membership_add_remove_group_with_vmm", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - // Create a project and a group - let (authz_project, ..) = - create_project(&opctx, &datastore, "my-project").await; - let group = create_anti_affinity_group( - &opctx, - &datastore, - &authz_project, - "my-group", - ) - .await - .unwrap(); - - // Also, create an affinity group. It'll be a member within the - // anti-affinity group. - let affinity_group = create_affinity_group( - &opctx, - &datastore, - &authz_project, - "my-affinity-group", - ) - .await - .unwrap(); - let affinity_group_id = - AffinityGroupUuid::from_untyped_uuid(affinity_group.id()); - - let (.., authz_aa_group) = LookupPath::new(opctx, datastore) - .anti_affinity_group_id(group.id()) - .lookup_for(authz::Action::Modify) - .await - .unwrap(); - let (.., authz_a_group) = LookupPath::new(opctx, datastore) - .affinity_group_id(affinity_group.id()) - .lookup_for(authz::Action::Modify) - .await - .unwrap(); - - // A new group should have no members - let pagparams = PaginatedBy::Id(DataPageParams { - marker: None, - limit: NonZeroU32::new(100).unwrap(), - direction: dropshot::PaginationOrder::Ascending, - }); - let members = datastore - .anti_affinity_group_member_list( - &opctx, - &authz_aa_group, - &pagparams, - ) - .await - .unwrap(); - assert!(members.is_empty()); - - // Create an instance without a VMM. - let instance = create_stopped_instance_record( - &opctx, - &datastore, - &authz_project, - "my-instance", - ) - .await; - - // Add the instance to the affinity group - datastore - .affinity_group_member_instance_add( - &opctx, - &authz_a_group, - instance, - ) - .await - .unwrap(); - - // Reserve the VMM for the instance. - allocate_instance_reservation(&datastore, instance).await; - - // Now, we cannot add the affinity group to the anti-affinity group - let err = datastore - .anti_affinity_group_member_affinity_group_add( - &opctx, - &authz_aa_group, - affinity_group_id, - ) - .await - .expect_err( - "Shouldn't be able to add running instances to anti-affinity groups", - ); - assert!(matches!(err, Error::InvalidRequest { .. })); - assert!( - err.to_string().contains( - "Affinity group with running instances cannot be added to an anti-affinity group" - ), - "{err:?}" - ); - - // If we have no reservation for the affinity group, we can add it to the - // anti-affinity group. - delete_instance_reservation(&datastore, instance).await; - datastore - .anti_affinity_group_member_affinity_group_add( - &opctx, - &authz_aa_group, - affinity_group_id, - ) - .await - .unwrap(); - - // Now we can reserve a sled for the instance once more. - allocate_instance_reservation(&datastore, instance).await; - - // We should now be able to list the new member - let members = datastore - .anti_affinity_group_member_list( - &opctx, - &authz_aa_group, - &pagparams, - ) - .await - .unwrap(); - assert_eq!(members.len(), 1); - assert!(matches!( - members[0], - external::AntiAffinityGroupMember::AffinityGroup { - id, - .. - } if id == affinity_group_id, - )); - - // We can delete the member and observe an empty member list -- even - // though it has an instance which is running! - datastore - .anti_affinity_group_member_affinity_group_delete( - &opctx, - &authz_aa_group, - affinity_group_id, - ) - .await - .unwrap(); - let members = datastore - .anti_affinity_group_member_list( - &opctx, - &authz_aa_group, - &pagparams, - ) - .await - .unwrap(); - assert!(members.is_empty()); - - // Clean up. - db.terminate().await; - logctx.cleanup_successful(); - } - #[tokio::test] async fn affinity_group_delete_group_deletes_members() { // Setup @@ -2936,15 +2447,6 @@ mod tests { .await .unwrap(); - let affinity_group = create_affinity_group( - &opctx, - &datastore, - &authz_project, - "my-affinity-group", - ) - .await - .unwrap(); - let (.., authz_aa_group) = LookupPath::new(opctx, datastore) .anti_affinity_group_id(group.id()) .lookup_for(authz::Action::Modify) @@ -2983,15 +2485,6 @@ mod tests { ) .await .unwrap(); - // Also, add the affinity member to the anti-affinity group as a member - datastore - .anti_affinity_group_member_affinity_group_add( - &opctx, - &authz_aa_group, - AffinityGroupUuid::from_untyped_uuid(affinity_group.id()), - ) - .await - .unwrap(); // Delete the group datastore @@ -3018,100 +2511,6 @@ mod tests { logctx.cleanup_successful(); } - // Since this name is gnarly, just to be clear: - // - Affinity groups can be "members" within anti-affinity groups - // - If one of these memberships is alive when the affinity group is - // deleted, that membership should be automatically removed - // - // Basically, do not keep around a reference to a dead affinity group. - #[tokio::test] - async fn affinity_group_delete_group_deletes_membership_in_anti_affinity_groups() - { - // Setup - let logctx = dev::test_setup_log( - "affinity_group_delete_group_deletes_membership_in_anti_affinity_groups", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - // Create a project and the groups - let (authz_project, ..) = - create_project(&opctx, &datastore, "my-project").await; - let affinity_group = create_affinity_group( - &opctx, - &datastore, - &authz_project, - "affinity", - ) - .await - .unwrap(); - let anti_affinity_group = create_anti_affinity_group( - &opctx, - &datastore, - &authz_project, - "anti-affinity", - ) - .await - .unwrap(); - - let (.., authz_a_group) = LookupPath::new(opctx, datastore) - .affinity_group_id(affinity_group.id()) - .lookup_for(authz::Action::Modify) - .await - .unwrap(); - let (.., authz_aa_group) = LookupPath::new(opctx, datastore) - .anti_affinity_group_id(anti_affinity_group.id()) - .lookup_for(authz::Action::Modify) - .await - .unwrap(); - - // Add the affinity group to the anti-affinity group. - let member = AffinityGroupUuid::from_untyped_uuid(affinity_group.id()); - datastore - .anti_affinity_group_member_affinity_group_add( - &opctx, - &authz_aa_group, - member, - ) - .await - .unwrap(); - - // Right now, the affinity group is observable - datastore - .anti_affinity_group_member_affinity_group_view( - &opctx, - &authz_aa_group, - member, - ) - .await - .expect("Group member should be visible - we just added it"); - - // Delete the affinity group (the member) - datastore.affinity_group_delete(&opctx, &authz_a_group).await.unwrap(); - - // The affinity group membership should have been revoked - let err = datastore - .anti_affinity_group_member_affinity_group_view( - &opctx, - &authz_aa_group, - member, - ) - .await - .expect_err("Group member should no longer exist"); - assert!( - matches!( - err, - Error::ObjectNotFound { type_name, .. } - if type_name == ResourceType::AntiAffinityGroupMember - ), - "Unexpected error: {err:?}" - ); - - // Clean up. - db.terminate().await; - logctx.cleanup_successful(); - } - #[tokio::test] async fn affinity_group_delete_instance_deletes_membership() { // Setup @@ -3435,60 +2834,17 @@ mod tests { let (authz_project, ..) = create_project(&opctx, &datastore, "my-project").await; - enum Member { - Instance, - AffinityGroup, - } - - impl Member { - fn resource_type(&self) -> ResourceType { - match self { - Member::Instance => ResourceType::Instance, - Member::AffinityGroup => ResourceType::AffinityGroup, - } - } - } - struct TestArgs { // Does the group exist? group: bool, - // What's the type of the member? - member_type: Member, // Does the member exist? member: bool, } let args = [ - TestArgs { - group: false, - member_type: Member::Instance, - member: false, - }, - TestArgs { - group: true, - member_type: Member::Instance, - member: false, - }, - TestArgs { - group: false, - member_type: Member::Instance, - member: true, - }, - TestArgs { - group: false, - member_type: Member::AffinityGroup, - member: false, - }, - TestArgs { - group: true, - member_type: Member::AffinityGroup, - member: false, - }, - TestArgs { - group: false, - member_type: Member::AffinityGroup, - member: true, - }, + TestArgs { group: false, member: false }, + TestArgs { group: true, member: false }, + TestArgs { group: false, member: true }, ]; for arg in args { @@ -3525,74 +2881,32 @@ mod tests { .await; let mut instance_exists = true; - // Create an affinity group - let affinity_group = create_affinity_group( - &opctx, - &datastore, - &authz_project, - "my-affinity-group", - ) - .await - .unwrap(); - let mut affinity_group_exists = true; - let (.., authz_instance) = LookupPath::new(opctx, datastore) .instance_id(instance.into_untyped_uuid()) .lookup_for(authz::Action::Modify) .await .unwrap(); - let (.., authz_affinity_group) = LookupPath::new(opctx, datastore) - .affinity_group_id(affinity_group.id()) - .lookup_for(authz::Action::Modify) - .await - .unwrap(); if !arg.member { - match arg.member_type { - Member::Instance => { - datastore - .project_delete_instance(&opctx, &authz_instance) - .await - .unwrap(); - instance_exists = false; - } - Member::AffinityGroup => { - datastore - .affinity_group_delete( - &opctx, - &authz_affinity_group, - ) - .await - .unwrap(); - affinity_group_exists = false; - } - } + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + instance_exists = false; } // Try to add the member to the group. // // Expect to see specific errors, depending on whether or not the // group/member exist. - let err = match arg.member_type { - Member::Instance => datastore - .anti_affinity_group_member_instance_add( - &opctx, - &authz_group, - instance, - ) - .await - .expect_err("Should have failed"), - Member::AffinityGroup => datastore - .anti_affinity_group_member_affinity_group_add( - &opctx, - &authz_group, - AffinityGroupUuid::from_untyped_uuid( - affinity_group.id(), - ), - ) - .await - .expect_err("Should have failed"), - }; + let err = datastore + .anti_affinity_group_member_instance_add( + &opctx, + &authz_group, + instance, + ) + .await + .expect_err("Should have failed"); match (arg.group, arg.member) { (false, _) => { @@ -3607,7 +2921,7 @@ mod tests { assert!( matches!(err, Error::ObjectNotFound { type_name, .. - } if type_name == arg.member_type.resource_type()), + } if type_name == ResourceType::Instance), "{err:?}" ); } @@ -3617,26 +2931,14 @@ mod tests { } // Do the same thing, but for group membership removal. - let err = match arg.member_type { - Member::Instance => datastore - .anti_affinity_group_member_instance_delete( - &opctx, - &authz_group, - instance, - ) - .await - .expect_err("Should have failed"), - Member::AffinityGroup => datastore - .anti_affinity_group_member_affinity_group_delete( - &opctx, - &authz_group, - AffinityGroupUuid::from_untyped_uuid( - affinity_group.id(), - ), - ) - .await - .expect_err("Should have failed"), - }; + let err = datastore + .anti_affinity_group_member_instance_delete( + &opctx, + &authz_group, + instance, + ) + .await + .expect_err("Should have failed"); match (arg.group, arg.member) { (false, _) => { @@ -3667,12 +2969,6 @@ mod tests { .await .unwrap(); } - if affinity_group_exists { - datastore - .affinity_group_delete(&opctx, &authz_affinity_group) - .await - .unwrap(); - } if arg.group { datastore .anti_affinity_group_delete(&opctx, &authz_group) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 1697e587863..c6ee5345c6e 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -1362,31 +1362,6 @@ pub(in crate::db::datastore) mod test { .unwrap(); } - async fn add_affinity_group_to_anti_affinity_group( - datastore: &DataStore, - aa_group_id: AntiAffinityGroupUuid, - a_group_id: AffinityGroupUuid, - ) { - use db::model::AntiAffinityGroupAffinityMembership; - use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; - - diesel::insert_into( - membership_dsl::anti_affinity_group_affinity_membership, - ) - .values(AntiAffinityGroupAffinityMembership::new( - aa_group_id, - a_group_id, - )) - .on_conflict(( - membership_dsl::anti_affinity_group_id, - membership_dsl::affinity_group_id, - )) - .do_nothing() - .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) - .await - .unwrap(); - } - // This short-circuits some of the logic and checks we normally have when // creating affinity groups, but makes testing easier. async fn add_instance_to_affinity_group( @@ -1490,42 +1465,6 @@ pub(in crate::db::datastore) mod test { Self { id_by_name } } - - async fn add_affinity_groups_to_anti_affinity_group( - &self, - datastore: &DataStore, - aa_group: &'static str, - a_groups: &[&'static str], - ) { - let (affinity, aa_group_id) = - self.id_by_name.get(aa_group).unwrap_or_else(|| { - panic!( - "Anti-affinity group {aa_group} not part of AllGroups" - ) - }); - assert!( - matches!(affinity, Affinity::Negative), - "{aa_group} should be an anti-affinity group, but is not" - ); - - for a_group in a_groups { - let (affinity, a_group_id) = - self.id_by_name.get(a_group).unwrap_or_else(|| { - panic!("Affinity group {a_group} not part of AllGroups") - }); - assert!( - matches!(affinity, Affinity::Positive), - "{a_group} should be an affinity group, but is not" - ); - - add_affinity_group_to_anti_affinity_group( - datastore, - AntiAffinityGroupUuid::from_untyped_uuid(*aa_group_id), - AffinityGroupUuid::from_untyped_uuid(*a_group_id), - ) - .await; - } - } } struct Instance { @@ -2240,333 +2179,6 @@ pub(in crate::db::datastore) mod test { logctx.cleanup_successful(); } - #[tokio::test] - async fn anti_affinity_group_containing_affinity_groups() { - let logctx = dev::test_setup_log( - "anti_affinity_group_containing_affinity_groups", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - let (authz_project, _project) = - create_project(&opctx, &datastore, "project").await; - - const SLED_COUNT: usize = 4; - let sleds = create_sleds(&datastore, SLED_COUNT).await; - - let groups = [ - Group { - affinity: Affinity::Negative, - name: "anti-affinity", - policy: external::AffinityPolicy::Fail, - }, - Group { - affinity: Affinity::Positive, - name: "affinity1", - policy: external::AffinityPolicy::Allow, - }, - Group { - affinity: Affinity::Positive, - name: "affinity2", - policy: external::AffinityPolicy::Allow, - }, - ]; - let all_groups = - AllGroups::create(&opctx, &datastore, &authz_project, &groups) - .await; - all_groups - .add_affinity_groups_to_anti_affinity_group( - &datastore, - "anti-affinity", - &["affinity1", "affinity2"], - ) - .await; - - // The instances on sled 0 and 1 belong to "affinity1" - // The instances on sled 2 and 3 belong to "affinity2" - // - // A new instance in "affinity2" can only be placed on sled 2. - let instances = [ - Instance::new().group("affinity1").sled(sleds[0].id()), - Instance::new().group("affinity1").sled(sleds[1].id()), - Instance::new().group("affinity2").sled(sleds[2].id()), - Instance::new() - .group("affinity2") - .sled(sleds[3].id()) - .use_many_resources(), - ]; - for instance in instances { - instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Failed to set up instances"); - } - - let test_instance = Instance::new().group("affinity2"); - let resource = test_instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Should have succeeded allocation"); - - assert_eq!( - resource.sled_id.into_untyped_uuid(), - sleds[2].id(), - "All sleds: {sled_ids:#?}", - sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn anti_affinity_group_containing_affinity_groups_force_away_from_affine() - { - let logctx = dev::test_setup_log( - "anti_affinity_group_containing_affinity_groups_force_away_from_affine", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - let (authz_project, _project) = - create_project(&opctx, &datastore, "project").await; - - const SLED_COUNT: usize = 5; - let sleds = create_sleds(&datastore, SLED_COUNT).await; - - let groups = [ - Group { - affinity: Affinity::Negative, - name: "anti-affinity", - policy: external::AffinityPolicy::Fail, - }, - Group { - affinity: Affinity::Positive, - name: "affinity1", - policy: external::AffinityPolicy::Allow, - }, - Group { - affinity: Affinity::Positive, - name: "affinity2", - policy: external::AffinityPolicy::Allow, - }, - ]; - let all_groups = - AllGroups::create(&opctx, &datastore, &authz_project, &groups) - .await; - all_groups - .add_affinity_groups_to_anti_affinity_group( - &datastore, - "anti-affinity", - &["affinity1", "affinity2"], - ) - .await; - - // The instances on sled 0 and 1 belong to "affinity1" - // The instance on sled 2 belongs to "affinity2" - // The instance on sled 3 directly belongs to "anti-affinity" - // - // Even though the instance belongs to the affinity groups - and - // would "prefer" to be co-located with them - it's forced to be placed - // on sleds[4], since that's the only sled which doesn't violate - // the hard "anti-affinity" requirement. - let instances = [ - Instance::new().group("affinity1").sled(sleds[0].id()), - Instance::new().group("affinity1").sled(sleds[1].id()), - Instance::new().group("affinity2").sled(sleds[2].id()), - Instance::new().group("anti-affinity").sled(sleds[3].id()), - ]; - for instance in instances { - instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Failed to set up instances"); - } - - // This instance is not part of "anti-affinity" directly. However, it - // is indirectly part of the anti-affinity group, because of its - // affinity group memberships. - let test_instance = - Instance::new().group("affinity1").group("affinity2"); - let resource = test_instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Should have succeeded allocation"); - - assert_eq!( - resource.sled_id.into_untyped_uuid(), - sleds[4].id(), - "All sleds: {sled_ids:#?}", - sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn anti_affinity_group_containing_affinity_groups_multigroup() { - let logctx = dev::test_setup_log( - "anti_affinity_group_containing_affinity_groups_multigroup", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - let (authz_project, _project) = - create_project(&opctx, &datastore, "project").await; - - const SLED_COUNT: usize = 3; - let sleds = create_sleds(&datastore, SLED_COUNT).await; - - let groups = [ - Group { - affinity: Affinity::Negative, - name: "wolf-eat-goat", - policy: external::AffinityPolicy::Fail, - }, - Group { - affinity: Affinity::Negative, - name: "goat-eat-cabbage", - policy: external::AffinityPolicy::Fail, - }, - Group { - affinity: Affinity::Positive, - name: "wolf", - policy: external::AffinityPolicy::Allow, - }, - Group { - affinity: Affinity::Positive, - name: "goat", - policy: external::AffinityPolicy::Allow, - }, - Group { - affinity: Affinity::Positive, - name: "cabbage", - policy: external::AffinityPolicy::Allow, - }, - ]; - let all_groups = - AllGroups::create(&opctx, &datastore, &authz_project, &groups) - .await; - all_groups - .add_affinity_groups_to_anti_affinity_group( - &datastore, - "wolf-eat-goat", - &["wolf", "goat"], - ) - .await; - all_groups - .add_affinity_groups_to_anti_affinity_group( - &datastore, - "goat-eat-cabbage", - &["goat", "cabbage"], - ) - .await; - - let instances = [ - Instance::new().group("wolf").sled(sleds[0].id()), - Instance::new().group("cabbage").sled(sleds[1].id()), - Instance::new().group("goat").sled(sleds[2].id()), - ]; - for instance in instances { - instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Failed to set up instances"); - } - - let test_instance = Instance::new().group("wolf").group("cabbage"); - let resource = test_instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Should have succeeded allocation"); - - // This instance can go on either sled[0] or sled[1], but not sled[2] - assert_ne!( - resource.sled_id.into_untyped_uuid(), - sleds[2].id(), - "All sleds: {sled_ids:#?}", - sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn anti_affinity_group_containing_overlapping_affinity_groups() { - let logctx = dev::test_setup_log( - "anti_affinity_group_containing_overlapping_affinity_groups", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - let (authz_project, _project) = - create_project(&opctx, &datastore, "project").await; - - const SLED_COUNT: usize = 4; - let sleds = create_sleds(&datastore, SLED_COUNT).await; - - let groups = [ - Group { - affinity: Affinity::Negative, - name: "anti-affinity", - policy: external::AffinityPolicy::Fail, - }, - Group { - affinity: Affinity::Positive, - name: "affinity1", - policy: external::AffinityPolicy::Allow, - }, - Group { - affinity: Affinity::Positive, - name: "affinity2", - policy: external::AffinityPolicy::Allow, - }, - ]; - let all_groups = - AllGroups::create(&opctx, &datastore, &authz_project, &groups) - .await; - all_groups - .add_affinity_groups_to_anti_affinity_group( - &datastore, - "anti-affinity", - &["affinity1", "affinity2"], - ) - .await; - - // The instances on sled 0 and 1 belong to "affinity1" - // The instances on sled 2 and 3 belong to "affinity2" - // - // If a new instance belongs to both affinity groups, it cannot satisfy - // the anti-affinity requirements. - let instances = [ - Instance::new().group("affinity1").sled(sleds[0].id()), - Instance::new().group("affinity1").sled(sleds[1].id()), - Instance::new().group("affinity2").sled(sleds[2].id()), - Instance::new().group("affinity2").sled(sleds[3].id()), - ]; - for instance in instances { - instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect("Failed to set up instances"); - } - - let test_instance = - Instance::new().group("affinity1").group("affinity2"); - let err = test_instance - .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) - .await - .expect_err("Should have failed to place instance"); - - let SledReservationTransactionError::Reservation( - SledReservationError::NotFound, - ) = err - else { - panic!("Unexpected error: {err:?}"); - }; - - db.terminate().await; - logctx.cleanup_successful(); - } - #[tokio::test] async fn affinity_multi_group() { let logctx = dev::test_setup_log("affinity_multi_group"); diff --git a/nexus/db-queries/src/db/queries/sled_reservation.rs b/nexus/db-queries/src/db/queries/sled_reservation.rs index 7fe28101a79..912fd644da4 100644 --- a/nexus/db-queries/src/db/queries/sled_reservation.rs +++ b/nexus/db-queries/src/db/queries/sled_reservation.rs @@ -57,70 +57,18 @@ pub fn sled_find_targets_query( COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " ).param().sql(" <= sled.reservoir_size ), - our_a_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_a_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_a_groups - ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE instance_id != ").param().sql(" - ), - our_direct_aa_groups AS ( - -- Anti-affinity groups to which our instance belongs + our_aa_groups AS ( SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = ").param().sql(" ), - other_direct_aa_instances AS ( + other_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id,instance_id FROM anti_affinity_group_instance_membership - JOIN our_direct_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id + JOIN our_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id WHERE instance_id != ").param().sql(" ), - our_indirect_aa_groups AS ( - -- Anti-affinity groups to which our instance's affinity groups belong - SELECT - anti_affinity_group_id, - affinity_group_id, - CASE - WHEN COUNT(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN TRUE - ELSE FALSE - END AS exactly_one_affinity_group - FROM anti_affinity_group_affinity_membership - WHERE affinity_group_id IN (SELECT group_id FROM our_a_groups) - ), - other_indirect_aa_instances_via_instances AS ( - SELECT anti_affinity_group_id AS group_id,instance_id - FROM anti_affinity_group_instance_membership - JOIN our_indirect_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_indirect_aa_groups.anti_affinity_group_id - ), - other_indirect_aa_instances_via_groups AS ( - SELECT anti_affinity_group_id AS group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_indirect_aa_groups - ON affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id - WHERE - -- If our instance belongs to exactly one of these groups... - CASE WHEN our_indirect_aa_groups.exactly_one_affinity_group - -- ... exclude that group from our anti-affinity - THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) - -- ... otherwise, consider all groups anti-affine - ELSE TRUE - END - ), - other_aa_instances AS ( - SELECT * FROM other_direct_aa_instances - UNION - SELECT * FROM other_indirect_aa_instances_via_instances - UNION - SELECT * FROM other_indirect_aa_instances_via_groups - ), other_aa_instances_by_policy AS ( SELECT policy,instance_id FROM other_aa_instances @@ -137,6 +85,18 @@ pub fn sled_find_targets_query( ON sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id ), + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), other_a_instances_by_policy AS ( SELECT policy,instance_id FROM other_a_instances @@ -212,70 +172,18 @@ pub fn sled_insert_resource_query( COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " ).param().sql(" <= sled.reservoir_size ), - our_a_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_a_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_a_groups - ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE instance_id != ").param().sql(" - ), - our_direct_aa_groups AS ( - -- Anti-affinity groups to which our instance belongs + our_aa_groups AS ( SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = ").param().sql(" ), - other_direct_aa_instances AS ( + other_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id,instance_id FROM anti_affinity_group_instance_membership - JOIN our_direct_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id + JOIN our_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id WHERE instance_id != ").param().sql(" ), - our_indirect_aa_groups AS ( - -- Anti-affinity groups to which our instance's affinity groups belong - SELECT - anti_affinity_group_id, - affinity_group_id, - CASE - WHEN COUNT(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN TRUE - ELSE FALSE - END AS only_one_affinity_group - FROM anti_affinity_group_affinity_membership - WHERE affinity_group_id IN (SELECT group_id FROM our_a_groups) - ), - other_indirect_aa_instances_via_instances AS ( - SELECT anti_affinity_group_id AS group_id,instance_id - FROM anti_affinity_group_instance_membership - JOIN our_indirect_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_indirect_aa_groups.anti_affinity_group_id - ), - other_indirect_aa_instances_via_groups AS ( - SELECT anti_affinity_group_id AS group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_indirect_aa_groups - ON affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id - WHERE - -- If our instance belongs to exactly one of these groups... - CASE WHEN our_indirect_aa_groups.only_one_affinity_group - -- ... exclude that group from our anti-affinity - THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) - -- ... otherwise, consider all groups anti-affine - ELSE TRUE - END - ), - other_aa_instances AS ( - SELECT * FROM other_direct_aa_instances - UNION - SELECT * FROM other_indirect_aa_instances_via_instances - UNION - SELECT * FROM other_indirect_aa_instances_via_groups - ), banned_instances AS ( SELECT instance_id FROM other_aa_instances @@ -293,6 +201,18 @@ pub fn sled_insert_resource_query( ON sled_resource_vmm.instance_id = banned_instances.instance_id ), + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), required_instances AS ( SELECT policy,instance_id FROM other_a_instances diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 10a73676ace..a94d31314d3 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -12,9 +12,6 @@ affinity_group_view GET /v1/affinity-groups/{affinity_ anti_affinity_group_create POST /v1/anti-affinity-groups anti_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group} anti_affinity_group_list GET /v1/anti-affinity-groups -anti_affinity_group_member_affinity_group_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} -anti_affinity_group_member_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} -anti_affinity_group_member_affinity_group_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group} anti_affinity_group_member_instance_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} anti_affinity_group_member_instance_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} anti_affinity_group_member_instance_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 4c2f0fdbb8c..b0f09288333 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -1449,42 +1449,6 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result; - /// Fetch anti-affinity group member of type affinity group - #[endpoint { - method = GET, - path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", - tags = ["affinity"], - }] - async fn anti_affinity_group_member_affinity_group_view( - rqctx: RequestContext, - query_params: Query, - path_params: Path, - ) -> Result, HttpError>; - - /// Add affinity group member to anti-affinity group - #[endpoint { - method = POST, - path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", - tags = ["affinity"], - }] - async fn anti_affinity_group_member_affinity_group_add( - rqctx: RequestContext, - query_params: Query, - path_params: Path, - ) -> Result, HttpError>; - - /// Remove affinity group member from anti-affinity group - #[endpoint { - method = DELETE, - path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", - tags = ["affinity"], - }] - async fn anti_affinity_group_member_affinity_group_delete( - rqctx: RequestContext, - query_params: Query, - path_params: Path, - ) -> Result; - /// Create anti-affinity group #[endpoint { method = POST, diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 35b161120be..daf94174c7f 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -22,7 +22,6 @@ use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::http_pagination::PaginatedBy; -use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; @@ -300,28 +299,6 @@ impl super::Nexus { .await } - pub(crate) async fn anti_affinity_group_member_affinity_group_view( - &self, - opctx: &OpContext, - anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, - affinity_group_lookup: &lookup::AffinityGroup<'_>, - ) -> Result { - let (.., authz_anti_affinity_group) = - anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; - let (.., authz_affinity_group) = - affinity_group_lookup.lookup_for(authz::Action::Read).await?; - let member = - AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()); - - self.db_datastore - .anti_affinity_group_member_affinity_group_view( - opctx, - &authz_anti_affinity_group, - member, - ) - .await - } - pub(crate) async fn affinity_group_member_add( &self, opctx: &OpContext, @@ -381,33 +358,6 @@ impl super::Nexus { }) } - pub(crate) async fn anti_affinity_group_member_affinity_group_add( - &self, - opctx: &OpContext, - anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, - affinity_group_lookup: &lookup::AffinityGroup<'_>, - ) -> Result { - let (.., authz_anti_affinity_group) = anti_affinity_group_lookup - .lookup_for(authz::Action::Modify) - .await?; - let (.., authz_affinity_group, affinity_group) = - affinity_group_lookup.fetch_for(authz::Action::Read).await?; - let member = - AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()); - - self.db_datastore - .anti_affinity_group_member_affinity_group_add( - opctx, - &authz_anti_affinity_group, - member, - ) - .await?; - Ok(external::AntiAffinityGroupMember::AffinityGroup { - id: member, - name: affinity_group.name().clone(), - }) - } - pub(crate) async fn affinity_group_member_delete( &self, opctx: &OpContext, @@ -450,27 +400,4 @@ impl super::Nexus { ) .await } - - pub(crate) async fn anti_affinity_group_member_affinity_group_delete( - &self, - opctx: &OpContext, - anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, - affinity_group_lookup: &lookup::AffinityGroup<'_>, - ) -> Result<(), Error> { - let (.., authz_anti_affinity_group) = anti_affinity_group_lookup - .lookup_for(authz::Action::Modify) - .await?; - let (.., authz_affinity_group) = - affinity_group_lookup.lookup_for(authz::Action::Read).await?; - let member = - AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()); - - self.db_datastore - .anti_affinity_group_member_affinity_group_delete( - opctx, - &authz_anti_affinity_group, - member, - ) - .await - } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 6365a60587d..19b4f8ec9bc 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -3092,142 +3092,6 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } - async fn anti_affinity_group_member_affinity_group_view( - rqctx: RequestContext, - query_params: Query, - path_params: Path, - ) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; - let query = query_params.into_inner(); - - // Select anti-affinity group - let group_selector = params::AntiAffinityGroupSelector { - anti_affinity_group: path.anti_affinity_group, - project: query.project.clone(), - }; - let group_lookup = - nexus.anti_affinity_group_lookup(&opctx, group_selector)?; - - // Select affinity group - let affinity_group_selector = params::AffinityGroupSelector { - project: query.project, - affinity_group: path.affinity_group, - }; - let affinity_group_lookup = - nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; - - let group = nexus - .anti_affinity_group_member_affinity_group_view( - &opctx, - &group_lookup, - &affinity_group_lookup, - ) - .await?; - - Ok(HttpResponseOk(group)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await - } - - async fn anti_affinity_group_member_affinity_group_add( - rqctx: RequestContext, - query_params: Query, - path_params: Path, - ) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - - // Select anti-affinity group - let group_selector = params::AntiAffinityGroupSelector { - anti_affinity_group: path.anti_affinity_group, - project: query.project.clone(), - }; - let group_lookup = - nexus.anti_affinity_group_lookup(&opctx, group_selector)?; - - // Select affinity group - let affinity_group_selector = params::AffinityGroupSelector { - project: query.project, - affinity_group: path.affinity_group, - }; - let affinity_group_lookup = - nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; - - let member = nexus - .anti_affinity_group_member_affinity_group_add( - &opctx, - &group_lookup, - &affinity_group_lookup, - ) - .await?; - Ok(HttpResponseCreated(member)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await - } - - async fn anti_affinity_group_member_affinity_group_delete( - rqctx: RequestContext, - query_params: Query, - path_params: Path, - ) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = - crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - - // Select anti-affinity group - let group_selector = params::AntiAffinityGroupSelector { - anti_affinity_group: path.anti_affinity_group, - project: query.project.clone(), - }; - let group_lookup = - nexus.anti_affinity_group_lookup(&opctx, group_selector)?; - - // Select affinity group - let affinity_group_selector = params::AffinityGroupSelector { - project: query.project, - affinity_group: path.affinity_group, - }; - let affinity_group_lookup = - nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; - - nexus - .anti_affinity_group_member_affinity_group_delete( - &opctx, - &group_lookup, - &affinity_group_lookup, - ) - .await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await - } - async fn anti_affinity_group_create( rqctx: RequestContext, query_params: Query, diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index cffa6bca717..672874aba35 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -518,16 +518,6 @@ pub static DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL: LazyLock = *DEMO_PROJECT_SELECTOR ) }); -pub static DEMO_ANTI_AFFINITY_GROUP_AFFINITY_GROUP_MEMBER_URL: LazyLock< - String, -> = LazyLock::new(|| { - format!( - "/v1/anti-affinity-groups/{}/members/affinity-group/{}?{}", - *DEMO_ANTI_AFFINITY_GROUP_NAME, - *DEMO_AFFINITY_GROUP_NAME, - *DEMO_PROJECT_SELECTOR - ) -}); pub static DEMO_ANTI_AFFINITY_GROUP_CREATE: LazyLock< params::AntiAffinityGroupCreate, @@ -2013,17 +2003,6 @@ pub static VERIFY_ENDPOINTS: LazyLock> = AllowedMethod::Post(serde_json::Value::Null), ], }, - VerifyEndpoint { - url: &DEMO_ANTI_AFFINITY_GROUP_AFFINITY_GROUP_MEMBER_URL, - visibility: Visibility::Protected, - unprivileged_access: UnprivilegedAccess::None, - - allowed_methods: vec![ - AllowedMethod::Get, - AllowedMethod::Delete, - AllowedMethod::Post(serde_json::Value::Null), - ], - }, VerifyEndpoint { url: &DEMO_IMPORT_DISK_BULK_WRITE_START_URL, visibility: Visibility::Protected, diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index d7c97ce4ca2..b15636618f3 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -323,12 +323,6 @@ static SETUP_REQUESTS: LazyLock> = LazyLock::new(|| { body: serde_json::Value::Null, id_routes: vec![], }, - // Add an affinity group member to the affinity group - SetupReq::Post { - url: &DEMO_ANTI_AFFINITY_GROUP_AFFINITY_GROUP_MEMBER_URL, - body: serde_json::Value::Null, - id_routes: vec![], - }, // Lookup the previously created NIC SetupReq::Get { url: &DEMO_INSTANCE_NIC_URL, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 91c2dd49735..90f5e171d5f 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -840,12 +840,6 @@ pub struct AntiAffinityInstanceGroupMemberPath { pub instance: NameOrId, } -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] -pub struct AntiAffinityAffinityGroupMemberPath { - pub anti_affinity_group: NameOrId, - pub affinity_group: NameOrId, -} - /// Create-time parameters for an `AntiAffinityGroup` #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct AntiAffinityGroupCreate { diff --git a/openapi/nexus.json b/openapi/nexus.json index e9c476438a0..4783e2fb56a 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -1487,154 +1487,6 @@ } } }, - "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}": { - "get": { - "tags": [ - "affinity" - ], - "summary": "Fetch anti-affinity group member of type affinity group", - "operationId": "anti_affinity_group_member_affinity_group_view", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "anti_affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupMember" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "post": { - "tags": [ - "affinity" - ], - "summary": "Add affinity group member to anti-affinity group", - "operationId": "anti_affinity_group_member_affinity_group_add", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "anti_affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AntiAffinityGroupMember" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "affinity" - ], - "summary": "Remove affinity group member from anti-affinity group", - "operationId": "anti_affinity_group_member_affinity_group_delete", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "path", - "name": "anti_affinity_group", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { "get": { "tags": [ @@ -12724,37 +12576,6 @@ "AntiAffinityGroupMember": { "description": "A member of an Anti-Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ - { - "description": "An affinity group belonging to this group", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "affinity_group" - ] - }, - "value": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "id", - "name" - ] - } - }, - "required": [ - "type", - "value" - ] - }, { "description": "An instance belonging to this group", "type": "object", @@ -23421,10 +23242,6 @@ } } }, - "TypedUuidForAffinityGroupKind": { - "type": "string", - "format": "uuid" - }, "TypedUuidForInstanceKind": { "type": "string", "format": "uuid" diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index fd344cd0b38..929788c2aaa 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4242,25 +4242,6 @@ CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_ins instance_id ); --- Describes an affinity group's membership within an anti-affinity group --- --- Since the naming here is a little confusing: --- This allows an anti-affinity group to contain affinity groups as members. --- This is useful for saying "I want these groups of VMMs to be anti-affine from --- one another", rather than "I want individual VMMs to be anti-affine from each other". -CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_affinity_membership ( - anti_affinity_group_id UUID NOT NULL, - affinity_group_id UUID NOT NULL, - - PRIMARY KEY (anti_affinity_group_id, affinity_group_id) -); - --- We need to look up all memberships of an affinity group so we can revoke these --- memberships efficiently when affinity groups are deleted -CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_affinity_membership_by_affinity_group ON omicron.public.anti_affinity_group_affinity_membership ( - affinity_group_id -); - -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, From af5c3fa4f94616f4c643ebb0ba2d532ad13b379d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 17 Mar 2025 17:52:57 -0700 Subject: [PATCH 81/84] shrinking the diff --- common/src/api/external/mod.rs | 4 +- nexus/db-queries/src/db/datastore/affinity.rs | 75 ++--- .../tests/output/sled_find_targets_query.sql | 80 +---- .../output/sled_insert_resource_query.sql | 80 +---- nexus/external-api/src/lib.rs | 6 +- nexus/tests/integration_tests/affinity.rs | 311 ++++++------------ nexus/tests/integration_tests/endpoints.rs | 1 - nexus/tests/integration_tests/unauthorized.rs | 4 +- 8 files changed, 180 insertions(+), 381 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 3d1b4330ba1..6a4464f228d 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1343,9 +1343,7 @@ impl SimpleIdentityOrName for AffinityGroupMember { /// /// Membership in a group is not exclusive - members may belong to multiple /// affinity / anti-affinity groups. -#[derive( - Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Hash, Eq, -)] +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { /// An instance belonging to this group diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index c0b53a924c8..53e1d08cb5b 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -232,18 +232,16 @@ impl DataStore { })?; // Ensure all memberships in the affinity group are deleted - { - use db::schema::affinity_group_instance_membership::dsl as member_dsl; - diesel::delete(member_dsl::affinity_group_instance_membership) - .filter(member_dsl::group_id.eq(authz_affinity_group.id())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; - } + use db::schema::affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; Ok(()) } @@ -321,19 +319,17 @@ impl DataStore { }) })?; - // Ensure all memberships in the anti-affinity group are deleted - { - use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; - diesel::delete(member_dsl::anti_affinity_group_instance_membership) - .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) - .execute_async(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel(e, ErrorHandler::Server) - }) - })?; - } + // Ensure all memberships in the anti affinity group are deleted + use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; Ok(()) } @@ -360,8 +356,7 @@ impl DataStore { .sql( " SELECT * FROM ( - SELECT - instance.id as id, + SELECT instance.id as id, instance.name as name, instance.state, instance.migration_id, @@ -2837,14 +2832,14 @@ mod tests { struct TestArgs { // Does the group exist? group: bool, - // Does the member exist? - member: bool, + // Does the instance exist? + instance: bool, } let args = [ - TestArgs { group: false, member: false }, - TestArgs { group: true, member: false }, - TestArgs { group: false, member: true }, + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, ]; for arg in args { @@ -2871,7 +2866,7 @@ mod tests { .unwrap(); } - // Create an instance + // Create an instance, and maybe deletes it let instance = create_stopped_instance_record( &opctx, &datastore, @@ -2879,7 +2874,6 @@ mod tests { "my-instance", ) .await; - let mut instance_exists = true; let (.., authz_instance) = LookupPath::new(opctx, datastore) .instance_id(instance.into_untyped_uuid()) @@ -2887,18 +2881,17 @@ mod tests { .await .unwrap(); - if !arg.member { + if !arg.instance { datastore .project_delete_instance(&opctx, &authz_instance) .await .unwrap(); - instance_exists = false; } - // Try to add the member to the group. + // Try to add the instnace to the group. // // Expect to see specific errors, depending on whether or not the - // group/member exist. + // group/instance exist. let err = datastore .anti_affinity_group_member_instance_add( &opctx, @@ -2908,7 +2901,7 @@ mod tests { .await .expect_err("Should have failed"); - match (arg.group, arg.member) { + match (arg.group, arg.instance) { (false, _) => { assert!( matches!(err, Error::ObjectNotFound { @@ -2940,7 +2933,7 @@ mod tests { .await .expect_err("Should have failed"); - match (arg.group, arg.member) { + match (arg.group, arg.instance) { (false, _) => { assert!( matches!(err, Error::ObjectNotFound { @@ -2963,7 +2956,7 @@ mod tests { } // Cleanup, if we actually created anything. - if instance_exists { + if arg.instance { datastore .project_delete_instance(&opctx, &authz_instance) .await diff --git a/nexus/db-queries/tests/output/sled_find_targets_query.sql b/nexus/db-queries/tests/output/sled_find_targets_query.sql index f230a4b6dcb..9de50f47f7f 100644 --- a/nexus/db-queries/tests/output/sled_find_targets_query.sql +++ b/nexus/db-queries/tests/output/sled_find_targets_query.sql @@ -17,75 +17,18 @@ WITH AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $3 <= sled.reservoir_size ), - our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $4), - other_a_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE - instance_id != $5 - ), - our_direct_aa_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $6), - other_direct_aa_instances + our_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $4), + other_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id, instance_id FROM anti_affinity_group_instance_membership - JOIN our_direct_aa_groups ON - anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id - WHERE - instance_id != $7 - ), - our_indirect_aa_groups - AS ( - SELECT - anti_affinity_group_id, - affinity_group_id, - CASE - WHEN count(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN true - ELSE false - END - AS exactly_one_affinity_group - FROM - anti_affinity_group_affinity_membership - WHERE - affinity_group_id IN (SELECT group_id FROM our_a_groups) - ), - other_indirect_aa_instances_via_instances - AS ( - SELECT - anti_affinity_group_id AS group_id, instance_id - FROM - anti_affinity_group_instance_membership - JOIN our_indirect_aa_groups ON - anti_affinity_group_instance_membership.group_id - = our_indirect_aa_groups.anti_affinity_group_id - ), - other_indirect_aa_instances_via_groups - AS ( - SELECT - anti_affinity_group_id AS group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_indirect_aa_groups ON - affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + JOIN our_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id WHERE - CASE - WHEN our_indirect_aa_groups.exactly_one_affinity_group - THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) - ELSE true - END - ), - other_aa_instances - AS ( - SELECT * FROM other_direct_aa_instances - UNION SELECT * FROM other_indirect_aa_instances_via_instances - UNION SELECT * FROM other_indirect_aa_instances_via_groups + instance_id != $5 ), other_aa_instances_by_policy AS ( @@ -108,6 +51,17 @@ WITH JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id ), + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $6), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $7 + ), other_a_instances_by_policy AS ( SELECT diff --git a/nexus/db-queries/tests/output/sled_insert_resource_query.sql b/nexus/db-queries/tests/output/sled_insert_resource_query.sql index a0ae17555dd..9cfb68e3008 100644 --- a/nexus/db-queries/tests/output/sled_insert_resource_query.sql +++ b/nexus/db-queries/tests/output/sled_insert_resource_query.sql @@ -20,75 +20,18 @@ WITH AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $4 <= sled.reservoir_size ), - our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $5), - other_a_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE - instance_id != $6 - ), - our_direct_aa_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $7), - other_direct_aa_instances + our_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $5), + other_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id, instance_id FROM anti_affinity_group_instance_membership - JOIN our_direct_aa_groups ON - anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id - WHERE - instance_id != $8 - ), - our_indirect_aa_groups - AS ( - SELECT - anti_affinity_group_id, - affinity_group_id, - CASE - WHEN count(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN true - ELSE false - END - AS only_one_affinity_group - FROM - anti_affinity_group_affinity_membership - WHERE - affinity_group_id IN (SELECT group_id FROM our_a_groups) - ), - other_indirect_aa_instances_via_instances - AS ( - SELECT - anti_affinity_group_id AS group_id, instance_id - FROM - anti_affinity_group_instance_membership - JOIN our_indirect_aa_groups ON - anti_affinity_group_instance_membership.group_id - = our_indirect_aa_groups.anti_affinity_group_id - ), - other_indirect_aa_instances_via_groups - AS ( - SELECT - anti_affinity_group_id AS group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_indirect_aa_groups ON - affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + JOIN our_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id WHERE - CASE - WHEN our_indirect_aa_groups.only_one_affinity_group - THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) - ELSE true - END - ), - other_aa_instances - AS ( - SELECT * FROM other_direct_aa_instances - UNION SELECT * FROM other_indirect_aa_instances_via_instances - UNION SELECT * FROM other_indirect_aa_instances_via_groups + instance_id != $6 ), banned_instances AS ( @@ -111,6 +54,17 @@ WITH banned_instances JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = banned_instances.instance_id ), + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $7), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $8 + ), required_instances AS ( SELECT diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index b0f09288333..e29dedf04a0 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -1413,7 +1413,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result>, HttpError>; - /// Fetch anti-affinity group member of type instance + /// Fetch anti-affinity group member #[endpoint { method = GET, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1425,7 +1425,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Add instance member to anti-affinity group + /// Add member to anti-affinity group #[endpoint { method = POST, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1437,7 +1437,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Remove instance member from anti-affinity group + /// Remove member from anti-affinity group #[endpoint { method = DELETE, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", diff --git a/nexus/tests/integration_tests/affinity.rs b/nexus/tests/integration_tests/affinity.rs index 556d9fb1608..debac93c0e1 100644 --- a/nexus/tests/integration_tests/affinity.rs +++ b/nexus/tests/integration_tests/affinity.rs @@ -162,60 +162,78 @@ impl ProjectScopedApiHelper<'_, T> { object_get_error(&self.client, &url, status).await } - async fn group_member_add( - &self, - group: &str, - member: &str, - ) -> T::Member { - let url = M::url(T::URL_COMPONENT, self.project, group, member); + async fn group_member_add(&self, group: &str, instance: &str) -> T::Member { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); object_create(&self.client, &url, &()).await } - async fn group_member_add_expect_error( + async fn group_member_add_expect_error( &self, group: &str, - member: &str, + instance: &str, status: StatusCode, ) -> HttpErrorResponseBody { - let url = M::url(T::URL_COMPONENT, self.project, group, member); + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); object_create_error(&self.client, &url, &(), status).await } - async fn group_member_get( - &self, - group: &str, - member: &str, - ) -> T::Member { - let url = M::url(T::URL_COMPONENT, self.project, group, member); + async fn group_member_get(&self, group: &str, instance: &str) -> T::Member { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); object_get(&self.client, &url).await } - async fn group_member_get_expect_error( + async fn group_member_get_expect_error( &self, group: &str, - member: &str, + instance: &str, status: StatusCode, ) -> HttpErrorResponseBody { - let url = M::url(T::URL_COMPONENT, self.project, group, member); + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); object_get_error(&self.client, &url, status).await } - async fn group_member_delete( - &self, - group: &str, - member: &str, - ) { - let url = M::url(T::URL_COMPONENT, self.project, group, member); + async fn group_member_delete(&self, group: &str, instance: &str) { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); object_delete(&self.client, &url).await } - async fn group_member_delete_expect_error( + async fn group_member_delete_expect_error( &self, group: &str, - member: &str, + instance: &str, status: StatusCode, ) -> HttpErrorResponseBody { - let url = M::url(T::URL_COMPONENT, self.project, group, member); + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); object_delete_error(&self.client, &url, status).await } } @@ -398,44 +416,14 @@ fn group_members_url(ty: &str, project: Option<&str>, group: &str) -> String { format!("/v1/{ty}/{group}/members{query_params}") } -/// Describes shared logic between "things that can be group members". -trait GroupMemberish { - fn url( - ty: &str, - project: Option<&str>, - group: &str, - member: &str, - ) -> String; -} - -struct MemberInstance {} - -impl GroupMemberish for MemberInstance { - fn url( - ty: &str, - project: Option<&str>, - group: &str, - member: &str, - ) -> String { - let query_params = project_query_param_suffix(project); - format!("/v1/{ty}/{group}/members/instance/{member}{query_params}") - } -} - -struct MemberAffinityGroup {} - -impl GroupMemberish for MemberAffinityGroup { - fn url( - ty: &str, - project: Option<&str>, - group: &str, - member: &str, - ) -> String { - let query_params = project_query_param_suffix(project); - format!( - "/v1/{ty}/{group}/members/affinity-group/{member}{query_params}" - ) - } +fn group_member_instance_url( + ty: &str, + project: Option<&str>, + group: &str, + instance: &str, +) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}/members/instance/{instance}{query_params}") } #[nexus_test(extra_sled_agents = 2)] @@ -496,10 +484,7 @@ async fn test_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { // Add these instances to an affinity group for instance in &instances { project_api - .group_member_add::( - GROUP_NAME, - &instance.identity.name.to_string(), - ) + .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) .await; } @@ -510,10 +495,7 @@ async fn test_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { // We can also list each member for instance in &instances { project_api - .group_member_get::( - GROUP_NAME, - instance.identity.name.as_str(), - ) + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) .await; } @@ -561,8 +543,8 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { const PROJECT_NAME: &'static str = "test-project"; const GROUP_NAME: &'static str = "group"; - const AFFINITY_GROUP_NAME: &'static str = "a-group"; const EXPECTED_SLEDS: usize = 3; + const INSTANCE_COUNT: usize = EXPECTED_SLEDS; let api = ApiHelper::new(external_client); @@ -580,118 +562,69 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { create_default_ip_pool(&external_client).await; api.create_project(PROJECT_NAME).await; - let aa_project_api = api.use_project::(PROJECT_NAME); - let a_project_api = api.use_project::(PROJECT_NAME); + let project_api = api.use_project::(PROJECT_NAME); - // Create both stopped instances and some affinity groups. - // - // All but two of the instances are going to be anti-affine from each other. - let mut aa_instances = Vec::new(); - for i in 0..EXPECTED_SLEDS - 1 { - aa_instances.push( - aa_project_api - .create_stopped_instance(&format!("test-aa-instance-{i}")) + let mut instances = Vec::new(); + for i in 0..INSTANCE_COUNT { + instances.push( + project_api + .create_stopped_instance(&format!("test-instance-{i}")) .await, ); } - // These two instances will be affine with each other, but anti-affine from - // everything else (indirectly) through their affinity group. - let mut a_instances = Vec::new(); - a_project_api.group_create(AFFINITY_GROUP_NAME).await; - for i in 0..2 { - let instance = a_project_api - .create_stopped_instance(&format!("test-a-instance-{i}")) - .await; - a_project_api - .group_member_add::( - AFFINITY_GROUP_NAME, - &instance.identity.name.to_string(), - ) - .await; - a_instances.push(instance); - } // When we start, we observe no anti-affinity groups - let groups = aa_project_api.groups_list().await; + let groups = project_api.groups_list().await; assert!(groups.is_empty()); // We can now create a group and observe it - let group = aa_project_api.group_create(GROUP_NAME).await; + let group = project_api.group_create(GROUP_NAME).await; // We can list it and also GET the group specifically - let groups = aa_project_api.groups_list().await; + let groups = project_api.groups_list().await; assert_eq!(groups.len(), 1); assert_eq!(groups[0].identity.id, group.identity.id); - let observed_group = aa_project_api.group_get(GROUP_NAME).await; + let observed_group = project_api.group_get(GROUP_NAME).await; assert_eq!(observed_group.identity.id, group.identity.id); // List all members of the anti-affinity group (expect nothing) - let members = aa_project_api.group_members_list(GROUP_NAME).await; + let members = project_api.group_members_list(GROUP_NAME).await; assert!(members.is_empty()); // Add these instances to the anti-affinity group - for instance in &aa_instances { - aa_project_api - .group_member_add::( - GROUP_NAME, - &instance.identity.name.to_string(), - ) + for instance in &instances { + project_api + .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) .await; } - // Add the affinity group to the anti-affinity group - aa_project_api - .group_member_add::( - GROUP_NAME, - AFFINITY_GROUP_NAME, - ) - .await; - // List members again (expect all instances, and the affinity group) - let members = aa_project_api.group_members_list(GROUP_NAME).await; - assert_eq!(members.len(), aa_instances.len() + 1); + // List members again (expect all instances) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), instances.len()); // We can also list each member - for instance in &aa_instances { - aa_project_api - .group_member_get::( - GROUP_NAME, - instance.identity.name.as_str(), - ) + for instance in &instances { + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) .await; } - aa_project_api - .group_member_get::( - GROUP_NAME, - AFFINITY_GROUP_NAME, - ) - .await; // Start the instances we created earlier. // // We don't actually care that they're "running" from the perspective of the // simulated sled agent, we just want placement to be triggered from Nexus. - for instance in &aa_instances { - api.start_instance(&instance).await; - } - for instance in &a_instances { + for instance in &instances { api.start_instance(&instance).await; } - let mut expected_aa_instances = aa_instances - .iter() - .map(|instance| instance.identity.id) - .collect::>(); - let mut expected_a_instances = a_instances + let mut expected_instances = instances .iter() .map(|instance| instance.identity.id) .collect::>(); // We expect that each sled will have a single instance, as all of the // instances will want to be anti-located from each other. - // - // The only exception will be a single sled containing both of the - // affine instances, which should be separate from all others. for sled in &sleds { let observed_instances = api .sled_instance_list(&sled.identity.id.to_string()) @@ -700,39 +633,22 @@ async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { .map(|sled_instance| sled_instance.identity.id) .collect::>(); - match observed_instances.len() { - 1 => { - // This should be one of the anti-affine instances - assert!( - expected_aa_instances.remove(&observed_instances[0]), - "The instance {} was observed too many times", - observed_instances[0] - ); - } - 2 => { - // This should be one of the affine instances, anti-affine from - // others because of their affinity groups - for observed_instance in observed_instances { - assert!( - expected_a_instances.remove(&observed_instance), - "The instance {} was observed too many times", - observed_instance - ); - } - } - _ => panic!( - "Unexpected instance count on sled: {observed_instances:?}" - ), - } + assert_eq!( + observed_instances.len(), + 1, + "All instances should be placed on distinct sleds" + ); + + assert!( + expected_instances.remove(&observed_instances[0]), + "The instance {} was observed on multiple sleds", + observed_instances[0] + ); } assert!( - expected_aa_instances.is_empty(), - "Did not find allocations for some anti-affine instances: {expected_aa_instances:?}" - ); - assert!( - expected_a_instances.is_empty(), - "Did not find allocations for some affine instances: {expected_a_instances:?}" + expected_instances.is_empty(), + "Did not find allocations for some instances: {expected_instances:?}" ); } @@ -786,11 +702,9 @@ async fn test_group_crud(client: &ClientTestContext) { // Add the instance to the affinity group let instance_name = &instance.identity.name.to_string(); - project_api - .group_member_add::(GROUP_NAME, &instance_name) - .await; + project_api.group_member_add(GROUP_NAME, &instance_name).await; let response = project_api - .group_member_add_expect_error::( + .group_member_add_expect_error( GROUP_NAME, &instance_name, StatusCode::BAD_REQUEST, @@ -809,18 +723,13 @@ async fn test_group_crud(client: &ClientTestContext) { let members = project_api.group_members_list(GROUP_NAME).await; assert_eq!(members.len(), 1); project_api - .group_member_get::( - GROUP_NAME, - instance.identity.name.as_str(), - ) + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) .await; // Delete the member, observe that it is gone + project_api.group_member_delete(GROUP_NAME, &instance_name).await; project_api - .group_member_delete::(GROUP_NAME, &instance_name) - .await; - project_api - .group_member_delete_expect_error::( + .group_member_delete_expect_error( GROUP_NAME, &instance_name, StatusCode::NOT_FOUND, @@ -829,7 +738,7 @@ async fn test_group_crud(client: &ClientTestContext) { let members = project_api.group_members_list(GROUP_NAME).await; assert_eq!(members.len(), 0); project_api - .group_member_get_expect_error::( + .group_member_get_expect_error( GROUP_NAME, &instance_name, StatusCode::NOT_FOUND, @@ -931,29 +840,21 @@ async fn test_group_project_selector( // Group Members can be added by name or UUID let instance_name = instance.identity.name.as_str(); let instance_id = instance.identity.id.to_string(); - project_api - .group_member_add::(GROUP_NAME, instance_name) - .await; - project_api - .group_member_delete::(GROUP_NAME, instance_name) - .await; - no_project_api - .group_member_add::(&group_id, &instance_id) - .await; - no_project_api - .group_member_delete::(&group_id, &instance_id) - .await; + project_api.group_member_add(GROUP_NAME, instance_name).await; + project_api.group_member_delete(GROUP_NAME, instance_name).await; + no_project_api.group_member_add(&group_id, &instance_id).await; + no_project_api.group_member_delete(&group_id, &instance_id).await; // Trying to use UUIDs with the project selector is invalid project_api - .group_member_add_expect_error::( + .group_member_add_expect_error( GROUP_NAME, &instance_id, StatusCode::BAD_REQUEST, ) .await; project_api - .group_member_add_expect_error::( + .group_member_add_expect_error( &group_id, instance_name, StatusCode::BAD_REQUEST, @@ -962,21 +863,21 @@ async fn test_group_project_selector( // Using any names without the project selector is invalid no_project_api - .group_member_add_expect_error::( + .group_member_add_expect_error( GROUP_NAME, &instance_id, StatusCode::BAD_REQUEST, ) .await; no_project_api - .group_member_add_expect_error::( + .group_member_add_expect_error( &group_id, instance_name, StatusCode::BAD_REQUEST, ) .await; no_project_api - .group_member_add_expect_error::( + .group_member_add_expect_error( GROUP_NAME, instance_name, StatusCode::BAD_REQUEST, diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 672874aba35..a606a0c94c2 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -518,7 +518,6 @@ pub static DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL: LazyLock = *DEMO_PROJECT_SELECTOR ) }); - pub static DEMO_ANTI_AFFINITY_GROUP_CREATE: LazyLock< params::AntiAffinityGroupCreate, > = LazyLock::new(|| params::AntiAffinityGroupCreate { diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index b15636618f3..5ccb0b2ceb1 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -304,7 +304,7 @@ static SETUP_REQUESTS: LazyLock> = LazyLock::new(|| { body: serde_json::to_value(&*DEMO_AFFINITY_GROUP_CREATE).unwrap(), id_routes: vec!["/v1/affinity-groups/{id}"], }, - // Add an instance member to the affinity group + // Add an instance to the affinity group SetupReq::Post { url: &DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL, body: serde_json::Value::Null, @@ -317,7 +317,7 @@ static SETUP_REQUESTS: LazyLock> = LazyLock::new(|| { .unwrap(), id_routes: vec!["/v1/anti-affinity-groups/{id}"], }, - // Add an instance member to the anti-affinity group + // Add an instance to the anti-affinity group SetupReq::Post { url: &DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL, body: serde_json::Value::Null, From 08801b1ef999fe6a4db5a8e804871ae5b5fe464c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 17 Mar 2025 18:00:15 -0700 Subject: [PATCH 82/84] Update nexus.json --- openapi/nexus.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openapi/nexus.json b/openapi/nexus.json index 4783e2fb56a..4a923b747f8 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -1492,7 +1492,7 @@ "tags": [ "affinity" ], - "summary": "Fetch anti-affinity group member of type instance", + "summary": "Fetch anti-affinity group member", "operationId": "anti_affinity_group_member_instance_view", "parameters": [ { @@ -1543,7 +1543,7 @@ "tags": [ "affinity" ], - "summary": "Add instance member to anti-affinity group", + "summary": "Add member to anti-affinity group", "operationId": "anti_affinity_group_member_instance_add", "parameters": [ { @@ -1594,7 +1594,7 @@ "tags": [ "affinity" ], - "summary": "Remove instance member from anti-affinity group", + "summary": "Remove member from anti-affinity group", "operationId": "anti_affinity_group_member_instance_delete", "parameters": [ { From 4e95b23f482eb218953a96817264200db13c0d1b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Mar 2025 12:29:03 -0700 Subject: [PATCH 83/84] Review feedback --- nexus/db-queries/src/db/datastore/affinity.rs | 42 +++-- nexus/db-queries/src/db/datastore/instance.rs | 148 ++++++++++-------- nexus/db-queries/src/db/datastore/mod.rs | 4 +- nexus/src/app/affinity.rs | 4 +- 4 files changed, 110 insertions(+), 88 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 53e1d08cb5b..b9f784acea1 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -10,7 +10,7 @@ use crate::authz::ApiResource; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::datastore::InstanceAndActiveVmm; +use crate::db::datastore::InstanceStateComputer; use crate::db::datastore::OpContext; use crate::db::error::ErrorHandler; use crate::db::error::public_error_from_diesel; @@ -439,10 +439,10 @@ impl DataStore { Ok(external::AffinityGroupMember::Instance { id: InstanceUuid::from_untyped_uuid(id), name: name.into(), - run_state: InstanceAndActiveVmm::determine_effective_state_inner( - instance_state, - migration_id, - vmm_state, + run_state: InstanceStateComputer::compute_state_from( + &instance_state, + migration_id.as_ref(), + vmm_state.as_ref(), ), }) }) @@ -556,10 +556,10 @@ impl DataStore { Ok(external::AntiAffinityGroupMember::Instance { id: InstanceUuid::from_untyped_uuid(id), name: name.into(), - run_state: InstanceAndActiveVmm::determine_effective_state_inner( - instance_state, - migration_id, - vmm_state, + run_state: InstanceStateComputer::compute_state_from( + &instance_state, + migration_id.as_ref(), + vmm_state.as_ref(), ), }) }) @@ -606,12 +606,11 @@ impl DataStore { )>(&*conn) .await .map(|(member, name, instance_state, migration_id, vmm_state)| { - let run_state = - InstanceAndActiveVmm::determine_effective_state_inner( - instance_state, - migration_id, - vmm_state, - ); + let run_state = InstanceStateComputer::compute_state_from( + &instance_state, + migration_id.as_ref(), + vmm_state.as_ref(), + ); member.to_external(name.into(), run_state) }) .map_err(|e| { @@ -665,12 +664,11 @@ impl DataStore { )>(&*conn) .await .map(|(member, name, instance_state, migration_id, vmm_state)| { - let run_state = - InstanceAndActiveVmm::determine_effective_state_inner( - instance_state, - migration_id, - vmm_state, - ); + let run_state = InstanceStateComputer::compute_state_from( + &instance_state, + migration_id.as_ref(), + vmm_state.as_ref(), + ); member.to_external(name.into(), run_state) }) .map_err(|e| { @@ -1926,7 +1924,7 @@ mod tests { } // Anti-affinity group member listing has a slightly more complicated - // implementation, because it queries multiple tables and UNIONs them + // implementation, because it queries multiple tables and JOINs them // together. // // This test exists to validate that manual implementation. diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 5aaa0f2f532..37cbc20c302 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -67,75 +67,28 @@ use omicron_uuid_kinds::SledUuid; use ref_cast::RefCast; use uuid::Uuid; -/// Wraps a record of an `Instance` along with its active `Vmm`, if it has one. -#[derive(Clone, Debug)] -pub struct InstanceAndActiveVmm { - pub instance: Instance, - pub vmm: Option, +pub struct InstanceStateComputer<'s> { + instance_state: &'s InstanceState, + migration_id: Option<&'s Uuid>, + vmm_state: Option<&'s VmmState>, } -impl InstanceAndActiveVmm { - pub fn instance(&self) -> &Instance { - &self.instance - } - - pub fn vmm(&self) -> &Option { - &self.vmm - } - - pub fn sled_id(&self) -> Option { - self.vmm.as_ref().map(|v| SledUuid::from_untyped_uuid(v.sled_id)) - } - - /// Returns the operator-visible [external API - /// `InstanceState`](external::InstanceState) for this instance and its - /// active VMM. - pub fn effective_state(&self) -> external::InstanceState { - Self::determine_effective_state(&self.instance, self.vmm.as_ref()) - } - - /// Returns the operator-visible [external API - /// `InstanceState`](external::InstanceState) for the provided [`Instance`] - /// and its active [`Vmm`], if one exists. - /// - /// # Arguments - /// - /// - `instance`: the instance - /// - `active_vmm`: the instance's active VMM, if one exists. - /// - /// # Notes - /// - /// Generally, the value of `active_vmm` should be - /// the VMM pointed to by `instance.runtime_state.propolis_id`. However, - /// this is not enforced by this function, as the `instance_migrate` saga - /// must in some cases determine an effective instance state from the - /// instance and *target* VMM states. - pub fn determine_effective_state( - instance: &Instance, - active_vmm: Option<&Vmm>, +impl<'s> InstanceStateComputer<'s> { + pub fn compute_state_from( + instance_state: &'s InstanceState, + migration_id: Option<&'s Uuid>, + vmm_state: Option<&'s VmmState>, ) -> external::InstanceState { - let instance_state = instance.runtime_state.nexus_state; - let migration_id = instance.runtime_state.migration_id; - let vmm_state = active_vmm.map(|vmm| vmm.runtime.state); - - Self::determine_effective_state_inner( - instance_state, - migration_id, - vmm_state, - ) + Self { instance_state, migration_id, vmm_state }.compute_state() } - pub fn determine_effective_state_inner( - instance_state: InstanceState, - migration_id: Option, - vmm_state: Option, - ) -> external::InstanceState { + fn compute_state(&self) -> external::InstanceState { use crate::db::model::InstanceState; use crate::db::model::VmmState; // We want to only report that an instance is `Stopped` when a new // `instance-start` saga is able to proceed. That means that: - match (instance_state, vmm_state) { + match (self.instance_state, self.vmm_state) { // - If there's an active migration ID for the instance, *always* // treat its state as "migration" regardless of the VMM's state. // @@ -158,7 +111,7 @@ impl InstanceAndActiveVmm { // instance-update saga will come along and remove the active VMM // and migration IDs. // - (InstanceState::Vmm, Some(_)) if migration_id.is_some() => { + (InstanceState::Vmm, Some(_)) if self.migration_id.is_some() => { external::InstanceState::Migrating } // - An instance with a "stopped" or "destroyed" VMM needs to be @@ -192,15 +145,84 @@ impl InstanceAndActiveVmm { // If there's a VMM state, and none of the above rules apply, use // that. (_instance_state, Some(vmm_state)) => { - debug_assert_eq!(_instance_state, InstanceState::Vmm); - vmm_state.into() + debug_assert_eq!(_instance_state, &InstanceState::Vmm); + (*vmm_state).into() } // If there's no VMM state, use the instance's state. - (instance_state, None) => instance_state.into(), + (instance_state, None) => (*instance_state).into(), + } + } +} + +impl<'s> From<&'s InstanceAndActiveVmm> for InstanceStateComputer<'s> { + fn from(i: &'s InstanceAndActiveVmm) -> Self { + Self { + instance_state: &i.instance.runtime_state.nexus_state, + migration_id: i.instance.runtime_state.migration_id.as_ref(), + vmm_state: i.vmm.as_ref().map(|vmm| &vmm.runtime.state), } } } +/// Wraps a record of an `Instance` along with its active `Vmm`, if it has one. +#[derive(Clone, Debug)] +pub struct InstanceAndActiveVmm { + pub instance: Instance, + pub vmm: Option, +} + +impl InstanceAndActiveVmm { + pub fn instance(&self) -> &Instance { + &self.instance + } + + pub fn vmm(&self) -> &Option { + &self.vmm + } + + pub fn sled_id(&self) -> Option { + self.vmm.as_ref().map(|v| SledUuid::from_untyped_uuid(v.sled_id)) + } + + /// Returns the operator-visible [external API + /// `InstanceState`](external::InstanceState) for this instance and its + /// active VMM. + pub fn effective_state(&self) -> external::InstanceState { + InstanceStateComputer::from(self).compute_state() + } + + /// Returns the operator-visible [external API + /// `InstanceState`](external::InstanceState) for the provided [`Instance`] + /// and its active [`Vmm`], if one exists. + /// + /// # Arguments + /// + /// - `instance`: the instance + /// - `active_vmm`: the instance's active VMM, if one exists. + /// + /// # Notes + /// + /// Generally, the value of `active_vmm` should be + /// the VMM pointed to by `instance.runtime_state.propolis_id`. However, + /// this is not enforced by this function, as the `instance_migrate` saga + /// must in some cases determine an effective instance state from the + /// instance and *target* VMM states. + pub fn determine_effective_state( + instance: &Instance, + active_vmm: Option<&Vmm>, + ) -> external::InstanceState { + let instance_state = instance.runtime_state.nexus_state; + let migration_id = instance.runtime_state.migration_id; + let vmm_state = active_vmm.map(|vmm| vmm.runtime.state); + + InstanceStateComputer::compute_state_from( + &instance_state, + migration_id.as_ref(), + vmm_state.as_ref(), + ) + } +} + impl From<(Instance, Option)> for InstanceAndActiveVmm { fn from(value: (Instance, Option)) -> Self { Self { instance: value.0, vmm: value.1 } diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 927526695fb..6d346a17af5 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -115,7 +115,9 @@ mod zpool; pub use address_lot::AddressLotCreateResult; pub use dns::DataStoreDnsTest; pub use dns::DnsVersionUpdateBuilder; -pub use instance::{InstanceAndActiveVmm, InstanceGestalt}; +pub use instance::{ + InstanceAndActiveVmm, InstanceGestalt, InstanceStateComputer, +}; pub use inventory::DataStoreInventoryTest; use nexus_db_model::AllSchemaVersions; pub use oximeter::CollectorReassignment; diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index daf94174c7f..e55795a19db 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -323,7 +323,7 @@ impl super::Nexus { name: instance.name().clone(), // TODO: This is kinda a lie - the current implementation of // "affinity_group_member_instance_add" relies on the instance - // note having a VMM, but that might change in the future. + // not having a VMM, but that might change in the future. run_state: external::InstanceState::Stopped, }) } @@ -353,7 +353,7 @@ impl super::Nexus { name: instance.name().clone(), // TODO: This is kinda a lie - the current implementation of // "anti_affinity_group_member_instance_add" relies on the instance - // note having a VMM, but that might change in the future. + // not having a VMM, but that might change in the future. run_state: external::InstanceState::Stopped, }) } From 72d0e9cb62bcc5905bf664f85bbfcf733ed15c90 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Mar 2025 13:30:35 -0700 Subject: [PATCH 84/84] more review feedback --- dev-tools/omdb/src/bin/omdb/db.rs | 8 +-- nexus/db-queries/src/db/datastore/instance.rs | 50 +++++-------------- nexus/src/app/instance.rs | 8 +-- 3 files changed, 21 insertions(+), 45 deletions(-) diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index e38f782a784..c9f7ef84da9 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -105,6 +105,7 @@ use nexus_db_queries::db::DataStore; use nexus_db_queries::db::datastore::CrucibleTargets; use nexus_db_queries::db::datastore::DataStoreConnection; use nexus_db_queries::db::datastore::InstanceAndActiveVmm; +use nexus_db_queries::db::datastore::InstanceStateComputer; use nexus_db_queries::db::datastore::read_only_resources_associated_with_volume; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; @@ -3567,10 +3568,9 @@ async fn cmd_db_instance_info( time_last_auto_restarted, } = instance.runtime_state; println!(" {STATE:>WIDTH$}: {nexus_state:?}"); - let effective_state = InstanceAndActiveVmm::determine_effective_state( - &instance, - active_vmm.as_ref(), - ); + let effective_state = + InstanceStateComputer::new(&instance, active_vmm.as_ref()) + .compute_state(); println!( "{} {API_STATE:>WIDTH$}: {effective_state:?}", if effective_state == InstanceState::Failed { "/!\\" } else { "(i)" } diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 37cbc20c302..7d2c458566e 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -67,6 +67,9 @@ use omicron_uuid_kinds::SledUuid; use ref_cast::RefCast; use uuid::Uuid; +/// Returns the operator-visible [external API +/// `InstanceState`](external::InstanceState) for the provided [`Instance`] +/// and its active [`Vmm`], if one exists. pub struct InstanceStateComputer<'s> { instance_state: &'s InstanceState, migration_id: Option<&'s Uuid>, @@ -74,6 +77,14 @@ pub struct InstanceStateComputer<'s> { } impl<'s> InstanceStateComputer<'s> { + pub fn new(instance: &'s Instance, vmm: Option<&'s Vmm>) -> Self { + Self { + instance_state: &instance.runtime_state.nexus_state, + migration_id: instance.runtime_state.migration_id.as_ref(), + vmm_state: vmm.as_ref().map(|vmm| &vmm.runtime.state), + } + } + pub fn compute_state_from( instance_state: &'s InstanceState, migration_id: Option<&'s Uuid>, @@ -82,7 +93,7 @@ impl<'s> InstanceStateComputer<'s> { Self { instance_state, migration_id, vmm_state }.compute_state() } - fn compute_state(&self) -> external::InstanceState { + pub fn compute_state(&self) -> external::InstanceState { use crate::db::model::InstanceState; use crate::db::model::VmmState; @@ -156,11 +167,7 @@ impl<'s> InstanceStateComputer<'s> { impl<'s> From<&'s InstanceAndActiveVmm> for InstanceStateComputer<'s> { fn from(i: &'s InstanceAndActiveVmm) -> Self { - Self { - instance_state: &i.instance.runtime_state.nexus_state, - migration_id: i.instance.runtime_state.migration_id.as_ref(), - vmm_state: i.vmm.as_ref().map(|vmm| &vmm.runtime.state), - } + InstanceStateComputer::new(&i.instance, i.vmm.as_ref()) } } @@ -190,37 +197,6 @@ impl InstanceAndActiveVmm { pub fn effective_state(&self) -> external::InstanceState { InstanceStateComputer::from(self).compute_state() } - - /// Returns the operator-visible [external API - /// `InstanceState`](external::InstanceState) for the provided [`Instance`] - /// and its active [`Vmm`], if one exists. - /// - /// # Arguments - /// - /// - `instance`: the instance - /// - `active_vmm`: the instance's active VMM, if one exists. - /// - /// # Notes - /// - /// Generally, the value of `active_vmm` should be - /// the VMM pointed to by `instance.runtime_state.propolis_id`. However, - /// this is not enforced by this function, as the `instance_migrate` saga - /// must in some cases determine an effective instance state from the - /// instance and *target* VMM states. - pub fn determine_effective_state( - instance: &Instance, - active_vmm: Option<&Vmm>, - ) -> external::InstanceState { - let instance_state = instance.runtime_state.nexus_state; - let migration_id = instance.runtime_state.migration_id; - let vmm_state = active_vmm.map(|vmm| vmm.runtime.state); - - InstanceStateComputer::compute_state_from( - &instance_state, - migration_id.as_ref(), - vmm_state.as_ref(), - ) - } } impl From<(Instance, Option)> for InstanceAndActiveVmm { diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 240419aa053..b0577aaf6e2 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -31,6 +31,7 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::DataStore; use nexus_db_queries::db::datastore::InstanceAndActiveVmm; +use nexus_db_queries::db::datastore::InstanceStateComputer; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; @@ -780,10 +781,9 @@ impl super::Nexus { vmm_state: &Option, requested: &InstanceStateChangeRequest, ) -> Result { - let effective_state = InstanceAndActiveVmm::determine_effective_state( - instance_state, - vmm_state.as_ref(), - ); + let effective_state = + InstanceStateComputer::new(instance_state, vmm_state.as_ref()) + .compute_state(); // Requests that operate on active instances have to be directed to the // instance's current sled agent. If there is none, the request needs to