test: garbage collection controller removes orphaned nics

.github/workflows/e2e-matrix.yaml

@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [Nonbehavioral, Utilization, GPU, Drift, Integration, NodeClaim, Chaos, ACR]
+        suite: [Nonbehavioral, Utilization, GPU, Drift, Integration, NodeClaim, Chaos, ACR, AzureGarbageCollection]
     permissions:
       contents: read
       id-token: write

.github/workflows/e2e.yaml

@@ -158,7 +158,7 @@ jobs:
         if: inputs.suite != 'Nonbehavioral'
         run: |
           AZURE_CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_RESOURCE_GROUP=${{ env.RG_NAME }} make az-creds
-          CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_ACR_NAME=${{ env.ACR_NAME}} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests
+          AZURE_LOCATION=${{ inputs.location }} AZURE_SUBSCRIPTION_ID=${{ secrets.E2E_SUBSCRIPTION_ID }} AZURE_RESOURCE_GROUP=${{ env.RG_NAME }} AZURE_CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_ACR_NAME=${{ env.ACR_NAME}} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests
       - name: dump logs on failure
         uses: ./.github/actions/e2e/dump-logs
         if: failure() || cancelled()

Makefile

@@ -49,7 +49,7 @@ e2etests: ## Run the e2e suite against your local cluster
 	# -count 1: prevents caching
 	# -timeout: If a test binary runs longer than TEST_TIMEOUT, panic
 	# -v: verbose output
-	cd test && CLUSTER_NAME=${CLUSTER_NAME} AZURE_ACR_NAME=${AZURE_ACR_NAME} go test \
+	cd test && AZURE_CLUSTER_NAME=${AZURE_CLUSTER_NAME} AZURE_ACR_NAME=${AZURE_ACR_NAME} AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP} AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID} AZURE_LOCATION=${AZURE_LOCATION} go test \
 		-p 1 \
 		-count 1 \
 		-timeout ${TEST_TIMEOUT} \

Makefile-az.mk

@@ -257,6 +257,7 @@ az-taintsystemnodes: ## Taint all system nodepool nodes
 az-taintnodes:
 	kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite
 
+
 az-e2etests: ## Run e2etests
 	kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite
 	TEST_SUITE=Utilization make e2etests

charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.17.1
+    controller-gen.kubebuilder.io/version: v0.17.2
   name: aksnodeclasses.karpenter.azure.com
 spec:
   group: karpenter.azure.com

go.mod

@@ -1,6 +1,6 @@
 module github.com/Azure/karpenter-provider-azure
 
-go 1.23.6
+go 1.23
 
 require (
 	github.com/Azure/azure-kusto-go v0.16.1

pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.17.1
+    controller-gen.kubebuilder.io/version: v0.17.2
   name: aksnodeclasses.karpenter.azure.com
 spec:
   group: karpenter.azure.com

test/pkg/environment/azure/environment.go

@@ -17,12 +17,16 @@ limitations under the License.
 package azure
 
 import (
+	"fmt"
+	"os"
 	"testing"
 
 	"github.com/samber/lo"
 	v1 "k8s.io/api/core/v1"
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
 	"github.com/Azure/karpenter-provider-azure/pkg/test"
 	"github.com/Azure/karpenter-provider-azure/test/pkg/environment/common"
@@ -40,16 +44,57 @@ const (
 
 type Environment struct {
 	*common.Environment
-	Region string
+	AzureClients
+	Vars
+}
+
+type Vars struct {
+	NodeResourceGroup    string
+	Region               string
+	SubscriptionID       string
+	VNETResourceGroup    string
+	ACRName              string
+	ClusterName          string
+	ClusterResourceGroup string
+}
+type AzureClients struct {
+	VNETClient       *armnetwork.VirtualNetworksClient
+	InterfacesClient *armnetwork.InterfacesClient
 }
 
 func NewEnvironment(t *testing.T) *Environment {
 	env := common.NewEnvironment(t)
-
-	return &Environment{
-		Region:      "westus2",
+	azureEnv := &Environment{
 		Environment: env,
+		Vars: Vars{
+			SubscriptionID:       os.Getenv("AZURE_SUBSCRIPTION_ID"),
+			ClusterName:          os.Getenv("AZURE_CLUSTER_NAME"),
+			ClusterResourceGroup: os.Getenv("AZURE_RESOURCE_GROUP"),
+			ACRName:              os.Getenv("AZURE_ACR_NAME"),
+			Region:               lo.Ternary(os.Getenv("AZURE_LOCATION") == "", "westus2", os.Getenv("AZURE_LOCATION")),
+		},
+	}
+
+	defaultNodeRG := fmt.Sprintf("MC_%s_%s_%s", azureEnv.ClusterResourceGroup, azureEnv.ClusterName, azureEnv.Region)
+	azureEnv.VNETResourceGroup = lo.Ternary(os.Getenv("VNET_RESOURCE_GROUP") == "", defaultNodeRG, os.Getenv("VNET_RESOURCE_GROUP"))
+	azureEnv.NodeResourceGroup = defaultNodeRG
+
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	interfacesClient, err := armnetwork.NewInterfacesClient(azureEnv.SubscriptionID, cred, nil)
+	if err != nil {
+		panic(err)
+	}
+	vnetClient, err := armnetwork.NewVirtualNetworksClient(azureEnv.SubscriptionID, cred, nil)
+	if err != nil {
+		panic(err)
 	}
+	azureEnv.VNETClient = vnetClient
+	azureEnv.InterfacesClient = interfacesClient
+	return azureEnv
 }
 
 func (env *Environment) DefaultAKSNodeClass() *v1alpha2.AKSNodeClass {

test/pkg/environment/azure/expectations.go

@@ -0,0 +1,83 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/samber/lo"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
+)
+
+func (env *Environment) EventuallyExpectKarpenterNicsToBeDeleted() {
+	GinkgoHelper()
+	Eventually(func() bool {
+		pager := env.InterfacesClient.NewListPager(env.NodeResourceGroup, nil)
+		for pager.More() {
+			resp, err := pager.NextPage(env.Context)
+			if err != nil {
+				return false
+			}
+
+			for _, nic := range resp.Value {
+				if nic.Tags != nil {
+					if _, exists := nic.Tags[strings.ReplaceAll(karpv1.NodePoolLabelKey, "/", "_")]; exists {
+						return false
+					}
+				}
+			}
+		}
+		return true
+	}).WithTimeout(10*time.Minute).WithPolling(10*time.Second).Should(BeTrue(), "Expected all orphan NICs to be deleted")
+}
+
+func (env *Environment) ExpectCreatedInterface(networkInterface armnetwork.Interface) {
+	GinkgoHelper()
+	poller, err := env.InterfacesClient.BeginCreateOrUpdate(env.Context, env.NodeResourceGroup, lo.FromPtr(networkInterface.Name), networkInterface, nil)
+	Expect(err).ToNot(HaveOccurred())
+	_, err = poller.PollUntilDone(env.Context, nil)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func (env *Environment) GetClusterSubnet() *armnetwork.Subnet {
+	GinkgoHelper()
+	vnet, err := getVNET(env.Context, env.VNETClient, env.VNETResourceGroup)
+	Expect(err).ToNot(HaveOccurred())
+	return vnet.Properties.Subnets[0]
+}
+
+func getVNET(ctx context.Context, client *armnetwork.VirtualNetworksClient, vnetRG string) (*armnetwork.VirtualNetwork, error) {
+	pager := client.NewListPager(vnetRG, nil)
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list virtual networks: %w", err)
+		}
+		if len(resp.VirtualNetworkListResult.Value) > 0 {
+			return resp.VirtualNetworkListResult.Value[0], nil
+		}
+	}
+	return nil, fmt.Errorf("no virtual networks found in resource group: %s", vnetRG)
+}

test/suites/acr/suite_test.go

@@ -18,7 +18,6 @@ package acr
 
 import (
 	"fmt"
-	"os"
 	"testing"
 	"time"
 
@@ -43,9 +42,7 @@ func TestAcr(t *testing.T) {
 	RegisterFailHandler(Fail)
 	BeforeSuite(func() {
 		env = azure.NewEnvironment(t)
-		acrName := os.Getenv("AZURE_ACR_NAME")
-		Expect(acrName).NotTo(BeEmpty(), "AZURE_ACR_NAME must be set for the acr test suite")
-		pauseImage = fmt.Sprintf("%s.azurecr.io/pause:3.6", acrName)
+		pauseImage = fmt.Sprintf("%s.azurecr.io/pause:3.6", env.ACRName)
 	})
 	RunSpecs(t, "Acr")
 }

test/suites/azuregarbagecollection/suite_test.go

@@ -0,0 +1,74 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuregarbagecollection
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
+	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	azkarptest "github.com/Azure/karpenter-provider-azure/pkg/test"
+	"github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure"
+	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
+)
+
+var env *azure.Environment
+var nodeClass *v1alpha2.AKSNodeClass
+var nodePool *karpv1.NodePool
+
+func TestGC(t *testing.T) {
+	RegisterFailHandler(Fail)
+	BeforeSuite(func() {
+		env = azure.NewEnvironment(t)
+	})
+	RunSpecs(t, "Azure Garbage Collection")
+}
+
+var _ = BeforeEach(func() {
+	env.BeforeEach()
+	nodeClass = env.DefaultAKSNodeClass()
+	nodePool = env.DefaultNodePool(nodeClass)
+})
+var _ = AfterEach(func() { env.Cleanup() })
+var _ = AfterEach(func() { env.AfterEach() })
+
+var _ = Describe("gc", func() {
+	It("should garbage collect network interfaces created by karpenter", func() {
+		env.ExpectCreatedInterface(armnetwork.Interface{
+			Name:     lo.ToPtr("orphan-nic"),
+			Location: lo.ToPtr(env.Region),
+			Tags:     azkarptest.ManagedTags("default"),
+			Properties: &armnetwork.InterfacePropertiesFormat{
+				IPConfigurations: []*armnetwork.InterfaceIPConfiguration{
+					{
+						Name: lo.ToPtr("ip-config"),
+						Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
+							Primary:                   lo.ToPtr(true),
+							Subnet:                    env.GetClusterSubnet(),
+							PrivateIPAllocationMethod: lo.ToPtr(armnetwork.IPAllocationMethodDynamic),
+						},
+					},
+				},
+			},
+		})
+		env.EventuallyExpectKarpenterNicsToBeDeleted()
+	})
+})