diff --git a/README.md b/README.md index 9246dd5c..59cb7f59 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This plugin is a very opinionated OpenShift wrapper designed to simplify the exe Executed with `kube-burner-ocp`, it looks like: ```console -$ kube-burner-ocp help +$ kube-burner-ocp --help kube-burner plugin designed to be used with OpenShift clusters as a quick way to run well-known workloads Usage: @@ -29,6 +29,8 @@ Available Commands: pvc-density Runs pvc-density workload udn-density-l3-pods Runs udn-density-l3-pods workload version Print the version number of kube-burner + virt-capacity-benchmark Runs capacity-benchmark workload + virt-density Runs virt-density workload web-burner-cluster-density Runs web-burner-cluster-density workload web-burner-init Runs web-burner-init workload web-burner-node-density Runs web-burner-node-density workload @@ -86,7 +88,7 @@ kube-burner-ocp cluster-density-v2 --iterations=1 --churn-duration=2m0s --churn- ### metrics-endpoints.yaml ```yaml -- endpoint: prometheus-k8s-openshift-monitoring.apps.rook.devshift.org +- endpoint: prometheus-k8s-openshift-monitoring.apps.rook.devshift.org metrics: - metrics.yml alerts: @@ -97,7 +99,7 @@ kube-burner-ocp cluster-density-v2 --iterations=1 --churn-duration=2m0s --churn- defaultIndex: {{.ES_INDEX}} type: opensearch - endpoint: https://prometheus-k8s-openshift-monitoring.apps.rook.devshift.org - token: {{ .TOKEN }} + token: {{ .TOKEN }} metrics: - metrics.yml indexer: @@ -387,6 +389,71 @@ Input parameters specific to the workload: | dpdk-cores | Number of cores assigned for each DPDK pod (should fill all the isolated cores of one NUMA node) | 2 | | performance-profile | Name of the performance profile implemented on the cluster | default | + +## Virt Workloads + +This workload family is a focused on Virtualization creating different objects across the cluster. + +The different variants are: +- [virt-density](#virt-density) +- [virt-capacity-benchmark](#virt-capacity-benchmark). + +### Virt Density + +### Virt Capacity Benchmark + +Test the capacity of Virtual Machines and Volumes supported by the cluster and a specific storage class. + +#### Environment Requirements + +In order to verify that the `VirtualMachine` completed their boot and that volume resize propagated successfully, the test uses `virtctl ssh`. +Therefore, `virtctl` must be installed and available in the `PATH`. + +See the [Temporary SSH Keys](#temporary-ssh-keys) for details on the SSH keys used for the test + +#### Test Sequence + +The test runs a workload in a loop without deleting previously created resources. By default it will continue until a failure occurs. +Each loop is comprised of the following steps: +- Create VMs +- Resize the root and data volumes +- Restart the VMs +- Snapshot the VMs +- Migrate the VMs + +#### Tested StorageClass + +By default, the test will search for the `StorageClass` to use: + +1. Use the default `StorageClass` for Virtualization annotated with `storageclass.kubevirt.io/is-default-virt-class` +2. If does not exist, use general default `StorageClass` annotated with `storageclass.kubernetes.io/is-default-class` +3. If does not exist, fail the test before starting + +To use a different one, use `--storage-class` to provide a different name. + +Please note that regardless to which `StorageClass` is used, it must: +- Support Volume Expansion: `allowVolumeExpansion: true`. +- Have a corresponding `VolumeSnapshotClass` using the same provisioner + +#### Test Namespace + +All `VirtualMachines` are created in the same namespace. + +By default, the namespace is `virt-capacity-benchmark`. Set it by passing `--namespace` (or `-n`) + +#### Test Size Parameters + +Users may control the workload sizes by passing the following arguments: +- `--max-iterations` - Maximum number of iterations, or 0 (default) for infinite. In any case, the test will stop upon failure +- `--vms` - Number of VMs for each iteration (default 5) +- `--data-volume-count` - Number of data volumes for each VM (default 9) + +#### Temporary SSH Keys + +The test generated the SSH keys automatically. +By default, it stores the pair in a temporary directory. +Users may choose the store the key in a specified directory by setting `--ssh-key-path` + ## Custom Workload: Bring your own workload To kickstart kube-burner-ocp with a custom workload, `init` becomes your go-to command. This command is equipped with flags that enable to seamlessly integrate and run your personalized workloads. Here's a breakdown of the flags accepted by the init command: diff --git a/cmd/config/virt-capacity-benchmark/check.sh b/cmd/config/virt-capacity-benchmark/check.sh new file mode 100755 index 00000000..e07d5574 --- /dev/null +++ b/cmd/config/virt-capacity-benchmark/check.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +COMMAND=$1 +LABEL_KEY=$2 +LABEL_VALUE=$3 +NAMESPACE=$4 +IDENTITY_FILE=$5 +REMOTE_USER=$6 +EXPECTED_ROOT_SIZE=$7 +EXPECTED_DATA_SIZE=$8 + +# Wait up to ~60 minutes +MAX_RETRIES=126 +# In the first reties use a shorter sleep +WAIT_TIMES=(1 2 3 5 10 20 30) + +if virtctl ssh --help | grep -qc "\--local-ssh " ; then + LOCAL_SSH="--local-ssh" +else + LOCAL_SSH="" +fi + +function get_vms() { + local namespace=$1 + local label_key=$2 + local label_value=$3 + + local vms + vms=$(kubectl get vm -n "${namespace}" -l "${label_key}"="${label_value}" -o json | jq .items | jq -r '.[] | .metadata.name') + local ret=$? + if [ $ret -ne 0 ]; then + echo "Failed to get VM list" + exit 1 + fi + echo "${vms}" +} + +function remote_command() { + local namespace=$1 + local identity_file=$2 + local remote_user=$3 + local vm_name=$4 + local command=$5 + + local output + output=$(virtctl ssh ${LOCAL_SSH} --local-ssh-opts="-o StrictHostKeyChecking=no" --local-ssh-opts="-o UserKnownHostsFile=/dev/null" -n "${namespace}" -i "${identity_file}" -c "${command}" --username "${remote_user}" "${vm_name}") + local ret=$? + if [ $ret -ne 0 ]; then + return 1 + fi + echo "${output}" +} + +function check_vm_running() { + remote_command "${NAMESPACE}" "${IDENTITY_FILE}" "${REMOTE_USER}" "${vm}" "ls" + return $? +} + +function check_resize() { + local vm=$1 + + local blk_devices + blk_devices=$(remote_command "${NAMESPACE}" "${IDENTITY_FILE}" "${REMOTE_USER}" "${vm}" "lsblk --json -v --output=NAME,SIZE") + local ret=$? + if [ $ret -ne 0 ]; then + return $ret + fi + + local size + size=$(echo "${blk_devices}" | jq .blockdevices | jq -r --arg name "vda" '.[] | select(.name == $name) | .size') + if [[ $size != "${EXPECTED_ROOT_SIZE}" ]]; then + return 1 + fi + + local datavolume_sizes + datavolume_sizes=$(echo "${blk_devices}" | jq .blockdevices | jq -r --arg name "vda" '.[] | select(.name != $name) | .size') + for datavolume_size in ${datavolume_sizes}; do + if [[ $datavolume_size != "${EXPECTED_DATA_SIZE}" ]]; then + return 1 + fi + done + + return 0 +} + +VMS=$(get_vms "${NAMESPACE}" "${LABEL_KEY}" "${LABEL_VALUE}") + +for vm in ${VMS}; do + counter=0 + for attempt in $(seq 1 $MAX_RETRIES); do + if ${COMMAND} "${vm}"; then + break + fi + if [ "${attempt}" -lt $MAX_RETRIES ]; then + if [ $counter -lt ${#WAIT_TIMES[@]} ]; then + wait_time=${WAIT_TIMES[$counter]} + else + wait_time=${WAIT_TIMES[-1]} # Use the last value in the array + fi + sleep "${wait_time}" + ((counter++)) + else + exit 1 + fi + done +done diff --git a/cmd/config/virt-capacity-benchmark/templates/resize_pvc.yml b/cmd/config/virt-capacity-benchmark/templates/resize_pvc.yml new file mode 100644 index 00000000..659823dd --- /dev/null +++ b/cmd/config/virt-capacity-benchmark/templates/resize_pvc.yml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +spec: + resources: + requests: + storage: {{ .storageSize }} diff --git a/cmd/config/virt-capacity-benchmark/templates/secret_ssh_public.yml b/cmd/config/virt-capacity-benchmark/templates/secret_ssh_public.yml new file mode 100644 index 00000000..f3eba27a --- /dev/null +++ b/cmd/config/virt-capacity-benchmark/templates/secret_ssh_public.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .name }}-{{ .counter }}" +type: Opaque +data: + key: {{ .publicKeyPath | ReadFile | b64enc }} \ No newline at end of file diff --git a/cmd/config/virt-capacity-benchmark/templates/vm-snapshot.yml b/cmd/config/virt-capacity-benchmark/templates/vm-snapshot.yml new file mode 100644 index 00000000..8ba51240 --- /dev/null +++ b/cmd/config/virt-capacity-benchmark/templates/vm-snapshot.yml @@ -0,0 +1,10 @@ +apiVersion: snapshot.kubevirt.io/v1beta1 +kind: VirtualMachineSnapshot +metadata: + name: "{{ .name }}-{{ .counter }}-{{ .Replica }}" +spec: + deletionPolicy: delete + source: + apiGroup: kubevirt.io + kind: VirtualMachine + name: "{{ .name }}-{{ .counter }}-{{ .Replica }}" diff --git a/cmd/config/virt-capacity-benchmark/templates/vm.yml b/cmd/config/virt-capacity-benchmark/templates/vm.yml new file mode 100644 index 00000000..8b422acc --- /dev/null +++ b/cmd/config/virt-capacity-benchmark/templates/vm.yml @@ -0,0 +1,101 @@ +{{- $storageClassName := .storageClassName -}} +{{- $dataVolumeLabels := .dataVolumeLabels -}} +{{- $dataVolumeSize := (default "1Gi" .dataVolumeSize) -}} +{{- $name := .name -}} +{{- $counter := .counter -}} +{{- $replica := .Replica }} + +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: "{{ $name }}-{{ $counter }}-{{ $replica }}" + labels: + {{range $key, $value := .vmLabels }} + {{ $key }}: {{ $value }} + {{end}} +spec: + dataVolumeTemplates: + - metadata: + name: "{{ $name }}-{{ $counter }}-{{ $replica }}-root" + labels: + {{range $key, $value := .rootVolumeLabels }} + {{ $key }}: {{ $value }} + {{end}} + spec: + source: + registry: + url: "docker://{{ .rootDiskImage }}" + storage: + storageClassName: {{ .storageClassName }} + resources: + requests: + storage: {{ default "10Gi" .rootVolumeSize }} + {{ range $dataVolumeIndex := .dataVolumeCounters }} + - metadata: + name: "{{ $name }}-{{ $counter }}-{{ $replica }}-data-{{ $dataVolumeIndex }}" + labels: + {{range $key, $value := $dataVolumeLabels }} + {{ $key }}: {{ $value }} + {{end}} + spec: + source: + blank: {} + storage: + storageClassName: {{ $storageClassName }} + resources: + requests: + storage: {{ $dataVolumeSize }} + {{ end }} + running: true + template: + spec: + accessCredentials: + - sshPublicKey: + propagationMethod: + noCloud: {} + source: + secret: + secretName: "{{ .sshPublicKeySecret }}-{{ .counter }}" + architecture: amd64 + domain: + resources: + requests: + memory: {{ default "512Mi" .vmMemory }} + devices: + disks: + - disk: + bus: virtio + name: rootdisk + bootOrder: 1 + {{ range $dataVolumeIndex := .dataVolumeCounters }} + - disk: + bus: virtio + name: "data-{{ $dataVolumeIndex }}" + {{ end }} + interfaces: + - name: default + masquerade: {} + bootOrder: 2 + machine: + type: pc-q35-rhel9.4.0 + networks: + - name: default + pod: {} + volumes: + - dataVolume: + name: "{{ .name }}-{{ .counter }}-{{ .Replica }}-root" + name: rootdisk + {{ range $dataVolumeIndex := .dataVolumeCounters }} + - dataVolume: + name: "{{ $name }}-{{ $counter }}-{{ $replica }}-data-{{ $dataVolumeIndex }}" + name: "data-{{ . }}" + {{ end }} + - cloudInitNoCloud: + userData: | + #cloud-config + chpasswd: + expire: false + password: {{ uuidv4 }} + user: fedora + runcmd: [] + name: cloudinitdisk diff --git a/cmd/config/virt-capacity-benchmark/virt-capacity-benchmark.yml b/cmd/config/virt-capacity-benchmark/virt-capacity-benchmark.yml new file mode 100644 index 00000000..f20a8c40 --- /dev/null +++ b/cmd/config/virt-capacity-benchmark/virt-capacity-benchmark.yml @@ -0,0 +1,184 @@ +{{- $kubeBurnerFQDN := "kube-burner.io" -}} +{{- $testName := "capacity-benchmark" }} +{{- $nsName := .testNamespace -}} +{{- $vmCount := .vmCount -}} +{{- $vmName := $testName -}} +{{- $sshPublicKeySecretName := $testName -}} +{{- $rootVolumeSize := 6 -}} +{{- $dataVolumeSize := 1 -}} +{{- $volumeLabelKey := (list $testName "." $kubeBurnerFQDN "/volume-type") | join "" -}} +{{- $volumeLabelValueRoot := "root" -}} +{{- $volumeLabelValueData := "data" -}} +{{- $jobCounterLabelKey := (list $testName "." $kubeBurnerFQDN "/counter") | join "" -}} +{{- $jobCounterLabelValue := (list "counter-" (.counter | toString )) | join "" -}} +{{- $testNamespacesLabelKey := (list $kubeBurnerFQDN "/test-name") | join "" -}} +{{- $testNamespacesLabelValue := $testName -}} +{{- $metricsBaseDirectory := $testName -}} +--- +global: + measurements: + - name: vmiLatency + +metricsEndpoints: +- indexer: + type: local + metricsDirectory: "./{{ $metricsBaseDirectory }}/iteration-{{ .counter | toString }}" + +jobs: +# Run cleanup only when counter is 0 +{{ if eq (.counter | int) 0 }} +- name: start-fresh + jobType: delete + waitForDeletion: true + qps: 5 + burst: 10 + objects: + - kind: Namespace + labelSelector: + {{ $testNamespacesLabelKey }}: {{ $testNamespacesLabelValue }} +{{ end }} + +- name: create-vms + jobType: create + jobIterations: 1 + qps: 20 + burst: 20 + namespacedIterations: false + namespace: {{ $nsName }} + namespaceLabels: + {{ $testNamespacesLabelKey }}: {{ $testNamespacesLabelValue }} + # verify object count after running each job + verifyObjects: true + errorOnVerify: true + # interval between jobs execution + jobIterationDelay: 20s + # wait all VMI be in the Ready Condition + waitWhenFinished: false + podWait: true + # timeout time after waiting for all object creation + maxWaitTimeout: 2h + jobPause: 10s + cleanup: false + # Set missing key as empty to allow using default values + defaultMissingKeysWithZero: true + beforeCleanup: "./check.sh check_vm_running {{ $jobCounterLabelKey }} {{ $jobCounterLabelValue }} {{ $nsName }} {{ .privateKey }} fedora" + objects: + + - objectTemplate: templates/secret_ssh_public.yml + runOnce: true + replicas: 1 + inputVars: + name: {{ $sshPublicKeySecretName }} + counter: {{ .counter | toString }} + publicKeyPath: {{ .publicKey }} + + - objectTemplate: templates/vm.yml + replicas: {{ $vmCount }} + waitOptions: + labelSelector: + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + inputVars: + name: {{ $vmName }} + counter: {{ .counter | toString }} + rootDiskImage: quay.io/containerdisks/fedora:latest + storageClassName: {{ .storageClassName }} + vmLabels: + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + rootVolumeLabels: + {{ $volumeLabelKey }}: {{ $volumeLabelValueRoot }} + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + rootVolumeSize: "{{ $rootVolumeSize | toString }}Gi" + dataVolumeSize: "{{ $dataVolumeSize | toString }}Gi" + dataVolumeLabels: + {{ $volumeLabelKey }}: {{ $volumeLabelValueData }} + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + sshPublicKeySecret: {{ $sshPublicKeySecretName }} + dataVolumeCounters: + {{ range .dataVolumeCounters }} + - {{ . }} + {{ end }} + +- name: resize-volumes + jobType: patch + jobIterations: 1 + jobIterationDelay: 15s + executionMode: sequential + qps: 20 + burst: 20 + beforeCleanup: "./check.sh check_resize {{ $jobCounterLabelKey }} {{ $jobCounterLabelValue }} {{ $nsName }} {{ .privateKey }} fedora 7G 2G" + objects: + - apiVersion: v1 + kind: PersistentVolumeClaim + labelSelector: + {{ $volumeLabelKey }}: {{ $volumeLabelValueData }} + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + patchType: "application/strategic-merge-patch+json" + objectTemplate: templates/resize_pvc.yml + inputVars: + storageSize: "{{ add $dataVolumeSize 1 | toString }}Gi" + - apiVersion: v1 + kind: PersistentVolumeClaim + labelSelector: + {{ $volumeLabelKey }}: {{ $volumeLabelValueRoot }} + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + patchType: "application/strategic-merge-patch+json" + objectTemplate: templates/resize_pvc.yml + inputVars: + storageSize: "{{ add $rootVolumeSize 1 | toString }}Gi" + +- name: restart-vms + jobType: kubevirt + qps: 20 + burst: 20 + jobIterations: 1 + maxWaitTimeout: 1h + objectDelay: 1m + beforeCleanup: "./check.sh check_vm_running {{ $jobCounterLabelKey }} {{ $jobCounterLabelValue }} {{ $nsName }} {{ .privateKey }} fedora" + objects: + - kubeVirtOp: restart + labelSelector: + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} + +- name: snapshot-vms + jobType: create + qps: 20 + burst: 20 + jobIterations: 1 + maxWaitTimeout: 1h + namespacedIterations: false + namespace: {{ $nsName }} + # verify object count after running each job + verifyObjects: true + errorOnVerify: true + # interval between jobs execution + jobIterationDelay: 20s + # wait all VMI be in the Ready Condition + waitWhenFinished: false + podWait: true + # timeout time after waiting for all object creation + jobPause: 10s + cleanup: false + # Set missing key as empty to allow using default values + defaultMissingKeysWithZero: true + preLoadImages: false + objects: + - objectTemplate: templates/vm-snapshot.yml + replicas: {{ $vmCount }} + inputVars: + name: {{ $vmName }} + counter: {{ .counter | toString }} + waitOptions: + forCondition: Ready + +- name: migrate-vms + jobType: kubevirt + qps: 20 + burst: 20 + jobIterations: 1 + maxWaitTimeout: 1h + objectDelay: 1m + beforeCleanup: "./check.sh check_vm_running {{ $jobCounterLabelKey }} {{ $jobCounterLabelValue }} {{ $nsName }} {{ .privateKey }} fedora" + objects: + - kubeVirtOp: migrate + labelSelector: + {{ $jobCounterLabelKey }}: {{ $jobCounterLabelValue }} diff --git a/cmd/ocp.go b/cmd/ocp.go index 7a1ed343..80d173cc 100644 --- a/cmd/ocp.go +++ b/cmd/ocp.go @@ -128,6 +128,7 @@ func openShiftCmd() *cobra.Command { ocp.NewVirtDensity(&wh), ocp.ClusterHealth(), ocp.CustomWorkload(&wh), + ocp.NewVirtCapacityBenchmark(&wh), ) util.SetupCmd(ocpCmd) return ocpCmd diff --git a/common.go b/common.go index 7621f734..d7aba7e5 100644 --- a/common.go +++ b/common.go @@ -15,15 +15,35 @@ package ocp import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" "encoding/json" + "encoding/pem" "fmt" "os" + "os/exec" + "path" "strings" ocpmetadata "github.com/cloud-bulldozer/go-commons/ocp-metadata" "github.com/kube-burner/kube-burner/pkg/config" "github.com/kube-burner/kube-burner/pkg/workloads" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/crypto/ssh" + kerrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +const ( + defaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class" + defaultVirtStorageClassAnnotation = "storageclass.kubevirt.io/is-default-virt-class" ) var clusterMetadata ocpmetadata.ClusterMetadata @@ -70,3 +90,175 @@ func GatherMetadata(wh *workloads.WorkloadHelper, alerting bool) error { } return nil } + +// GenerateSSHKeyPair generates an SSH key pair and saves them to the specified files +func generateSSHKeyPair(sshKeyPairPath string) (string, string, error) { + if sshKeyPairPath == "" { + tempDir, err := os.MkdirTemp("", tmpDirPattern) + if err != nil { + log.Fatalln("Error creating temporary directory:", err) + } + sshKeyPairPath = tempDir + } + privateKeyPath := path.Join(sshKeyPairPath, sshKeyFileName) + publicKeyPath := path.Join(sshKeyPairPath, strings.Join([]string{sshKeyFileName, "pub"}, ".")) + + // Generate RSA private key + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", "", fmt.Errorf("failed to generate private key: %w", err) + } + + // Encode the private key to PEM format + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }) + + // Write the private key to a file + err = os.WriteFile(privateKeyPath, privateKeyPEM, 0600) + if err != nil { + return "", "", fmt.Errorf("failed to write private key to file: %w", err) + } + + // Generate the public key in OpenSSH authorized_keys format + publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return "", "", fmt.Errorf("failed to generate public key: %w", err) + } + publicKeyBytes := ssh.MarshalAuthorizedKey(publicKey) + + // Write the public key to a file + err = os.WriteFile(publicKeyPath, publicKeyBytes, 0644) + if err != nil { + return "", "", fmt.Errorf("failed to write public key to file: %w", err) + } + + log.Infof("SSH keys saved to [%s]", sshKeyPairPath) + + return privateKeyPath, publicKeyPath, nil +} + +func generateLoopCounterSlice(length int) []string { + counter := make([]string, length) + for i := 0; i < length; i++ { + counter[i] = fmt.Sprint(i + 1) + } + return counter +} + +// If storageClassName was provided, verify that it exists. Otherwise, get the default +func getStorageClassName(storageClassName string, preferVirt bool) (string, error) { + kubeClientProvider := config.NewKubeClientProvider("", "") + clientSet, _ := kubeClientProvider.ClientSet(0, 0) + + if storageClassName != "" { + if err := storageClassExists(clientSet, storageClassName); err != nil { + return "", err + } + return storageClassName, nil + } + return getDefaultStorageClassName(clientSet, preferVirt) +} + +func storageClassExists(clientSet kubernetes.Interface, storageClassName string) error { + _, err := clientSet.StorageV1().StorageClasses().Get(context.Background(), storageClassName, v1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return fmt.Errorf("storageClass %s does not exist", storageClassName) + } + return fmt.Errorf("failed to verify the StorageClass %s - %v", storageClassName, err) + } + return nil +} + +func getDefaultStorageClassName(clientSet kubernetes.Interface, preferVirt bool) (string, error) { + storageClasses, err := clientSet.StorageV1().StorageClasses().List(context.Background(), v1.ListOptions{}) + if err != nil { + return "", fmt.Errorf("failed to list StorageClass objects - %v", err) + } + + var defaultStorageClassName, defaultVirtStorageClassName, storageClassName string + for _, storageClass := range storageClasses.Items { + if val, ok := storageClass.Annotations[defaultStorageClassAnnotation]; ok && val == "true" { + defaultStorageClassName = storageClass.GetName() + } + if val, ok := storageClass.Annotations[defaultVirtStorageClassAnnotation]; ok && val == "true" { + defaultVirtStorageClassName = storageClass.GetName() + } + } + if preferVirt { + storageClassName = defaultVirtStorageClassName + } + if storageClassName == "" { + storageClassName = defaultStorageClassName + } + if storageClassName == "" { + return "", fmt.Errorf("storageClassName was not provided with no default StorageClass") + } + return storageClassName, nil +} + +func storageClassSupportsVolumeExpansion(storageClassName string) bool { + kubeClientProvider := config.NewKubeClientProvider("", "") + clientSet, _ := kubeClientProvider.ClientSet(0, 0) + + storageClass, err := clientSet.StorageV1().StorageClasses().Get(context.Background(), storageClassName, v1.GetOptions{}) + if err != nil { + log.Errorf("Failed to get storageClass resource - %v", err) + return false + } + + return storageClass.AllowVolumeExpansion != nil && *storageClass.AllowVolumeExpansion +} + +func getStorageClassProvisioner(storageClassName string) (string, error) { + kubeClientProvider := config.NewKubeClientProvider("", "") + clientSet, _ := kubeClientProvider.ClientSet(0, 0) + + storageClass, err := clientSet.StorageV1().StorageClasses().Get(context.Background(), storageClassName, v1.GetOptions{}) + if err != nil { + return "", err + } + + return storageClass.Provisioner, nil +} + +func getDynamicClient() *dynamic.DynamicClient { + _, restConfig := config.NewKubeClientProvider("", "").ClientSet(0, 0) + return dynamic.NewForConfigOrDie(restConfig) +} + +func getVolumeSnapshotClassNameForProvisioner(provisioner string) (string, error) { + volumeSnapshotClassGVR := schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshotclasses"} + client := getDynamicClient() + itemList, err := client.Resource(volumeSnapshotClassGVR).List(context.TODO(), v1.ListOptions{}) + if err != nil { + return "", fmt.Errorf("failed listing %s: %v", volumeSnapshotClassGVR.Resource, err) + } + var snapshotClassName string + for _, item := range itemList.Items { + driver, found, err := unstructured.NestedString(item.Object, "driver") + if err != nil || !found { + return "", fmt.Errorf("failed to get driver for %s/%s - %v", volumeSnapshotClassGVR.Resource, item.GetName(), err) + } + if driver == provisioner { + snapshotClassName = item.GetName() + break + } + } + return snapshotClassName, nil +} + +func getVolumeSnapshotClassNameForStorageClass(storageClassName string) (string, error) { + provisioner, err := getStorageClassProvisioner(storageClassName) + if err != nil { + return "", err + } + return getVolumeSnapshotClassNameForProvisioner(provisioner) +} + +func virtctlExists() bool { + err := exec.Command("virtctl", "version").Run() + return err == nil +} diff --git a/go.mod b/go.mod index 53ac4960..edfd2690 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/praserx/ipconv v1.2.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 + golang.org/x/crypto v0.31.0 k8s.io/apimachinery v0.31.1 k8s.io/client-go v0.31.1 ) @@ -71,7 +72,6 @@ require ( github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect diff --git a/virt-capacity-benchmark.go b/virt-capacity-benchmark.go new file mode 100644 index 00000000..d3d454f1 --- /dev/null +++ b/virt-capacity-benchmark.go @@ -0,0 +1,114 @@ +// Copyright 2025 The Kube-burner Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ocp + +import ( + "fmt" + "os" + + "github.com/kube-burner/kube-burner/pkg/workloads" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +const ( + sshKeyFileName = "ssh" + tmpDirPattern = "kube-burner-capacity-benchmark-*" + testName = "virt-capacity-benchmark" +) + +// NewVirtCapacityBenchmark holds the virt-capacity-benchmark workload +func NewVirtCapacityBenchmark(wh *workloads.WorkloadHelper) *cobra.Command { + var storageClassName string + var sshKeyPairPath string + var maxIterations int + var vmsPerIteration int + var dataVolumeCount int + var testNamespace string + var metricsProfiles []string + var rc int + cmd := &cobra.Command{ + Use: testName, + Short: "Runs capacity-benchmark workload", + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, args []string) { + var err error + + if !virtctlExists() { + log.Fatalf("Failed to run virtctl. Check that it is installed, in PATH and working") + } + + storageClassName, err = getStorageClassName(storageClassName, true) + if err != nil { + log.Fatal(err) + } + if !storageClassSupportsVolumeExpansion(storageClassName) { + log.Fatalf("Storage Class [%s] does not support volume expansion", storageClassName) + } + volumeSnapshotClassName, err := getVolumeSnapshotClassNameForStorageClass(storageClassName) + if err != nil { + log.Fatalf("Failed to get VolumeSnapshotClass for StorageClass %s - %v", storageClassName, err) + } + if volumeSnapshotClassName == "" { + log.Fatalf("Could not find a corresponding VolumeSnapshotClass for StorageClass %s", storageClassName) + } + log.Infof("Running tests with Storage Class [%s]", storageClassName) + }, + Run: func(cmd *cobra.Command, args []string) { + privateKeyPath, publicKeyPath, err := generateSSHKeyPair(sshKeyPairPath) + if err != nil { + log.Fatalf("Failed to generate SSH keys for the test - %v", err) + } + + additionalVars := map[string]interface{}{ + "privateKey": privateKeyPath, + "publicKey": publicKeyPath, + "vmCount": fmt.Sprint(vmsPerIteration), + "storageClassName": storageClassName, + "testNamespace": testNamespace, + "dataVolumeCounters": generateLoopCounterSlice(dataVolumeCount), + } + + setMetrics(cmd, metricsProfiles) + + log.Infof("Running tests in Namespace [%s]", testNamespace) + counter := 0 + for { + os.Setenv("counter", fmt.Sprint(counter)) + rc = wh.RunWithAdditionalVars(cmd.Name(), additionalVars) + if rc != 0 { + log.Infof("Capacity failed in loop #%d", counter) + break + } + counter += 1 + if maxIterations > 0 && counter >= maxIterations { + log.Infof("Reached maxIterations [%d]", maxIterations) + break + } + } + }, + PostRun: func(cmd *cobra.Command, args []string) { + os.Exit(rc) + }, + } + cmd.Flags().StringVar(&storageClassName, "storage-class", "", "Name of the Storage Class to test") + cmd.Flags().StringVar(&sshKeyPairPath, "ssh-key-path", "", "Path to save the generarated SSH keys - default to a temporary location") + cmd.Flags().IntVar(&maxIterations, "max-iterations", 0, "Maximum times to run the test sequence. Default - run until failure (0)") + cmd.Flags().IntVar(&vmsPerIteration, "vms", 5, "Number of VMs to test in each iteration") + cmd.Flags().IntVar(&dataVolumeCount, "data-volume-count", 9, "Number of data volumes per VM ") + cmd.Flags().StringVarP(&testNamespace, "namespace", "n", testName, "Namespace to run the test in") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-aggregated.yml"}, "Comma separated list of metrics profiles to use") + return cmd +}