test: use kyverno/chainsaw for E2E testing (#642)

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com>

Author: odubajDT

.chainsaw-local.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: configuration-local
+spec:
+  template: true
+  timeouts:
+    assert: 2m
+    error: 2m

.chainsaw.yaml

@@ -0,0 +1,11 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: configuration
+spec:
+  skipDelete: true
+  template: true
+  timeouts:
+    assert: 2m
+    error: 2m

.github/workflows/e2e.yml

@@ -0,0 +1,70 @@
+name: E2E-Test
+on:
+  workflow_call:
+    inputs:
+      kind_tag:
+        description: "Tag of Kind image"
+        type: "string"
+env:
+  GO_VERSION: "~1.21"
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  e2e-test:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: open-feature-operator-local-${{ github.sha }}
+          path: ${{ github.workspace }}
+
+      - name: Cache build tools
+        id: cache-build-tools
+        uses: actions/cache@v4
+        with:
+          path: ./bin
+          key: build-tools-${{ github.ref_name }}
+
+      - name: Load open-feature-operator image into docker
+        run: |
+          docker load --input ${{ github.workspace }}/open-feature-operator-local.tar
+
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.9.0
+        with:
+          config: ./test/e2e/kind-cluster.yml
+          cluster_name: open-feature-operator-test
+          node_image: kindest/node:${{ inputs.kind_tag }}
+
+      - name: Load open-feature-operator image into Kind cluster
+        run: |
+          kind load docker-image open-feature-operator-local:${{ github.sha }} --name open-feature-operator-test
+
+      - name: Install Chainsaw
+        uses: kyverno/action-install-chainsaw@748066cc1580718e7924c9b689a8d366bde05100 # v0.2.0
+
+      - name: Install OFO
+        run: |
+          IMG=open-feature-operator-local:${{ github.sha }} make deploy-operator
+
+      - name: Run e2e test
+        run: |
+          make e2e-test-chainsaw
+
+      - name: Create reports
+        if: failure()
+        working-directory: ./.github/scripts
+        run: ./create-reports.sh
+
+      - name: Upload cluster logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: e2e-tests
+          path: .github/scripts/logs

.github/workflows/pr-checks.yml

@@ -12,8 +12,6 @@ on:
 env:
   # Default minimum version of Go to support.
   DEFAULT_GO_VERSION: "~1.21"
-  # renovate: datasource=github-tags depName=kudobuilder/kuttl
-  KUTTL_VERSION: "v0.15.0"
 
 permissions:
   contents: read
@@ -97,62 +95,13 @@ jobs:
           name: open-feature-operator-local-${{ github.sha }}
           path: ${{ github.workspace }}/open-feature-operator-local.tar
 
-  e2e-test:
-    runs-on: ubuntu-22.04
-    needs: docker-local
+  e2e_tests:
+    name: E2E Tests
+    needs:
+      - docker-local
     strategy:
       matrix:
-        node_image: [ v1.22.17, v1.23.17, v1.24.15, v1.25.11, v1.26.3, v1.27.3, v1.28.0 ]
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.DEFAULT_GO_VERSION }}
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Download image
-        uses: actions/download-artifact@v3
-        with:
-          name: open-feature-operator-local-${{ github.sha }}
-          path: ${{ github.workspace }}
-
-      - name: Cache build tools
-        id: cache-build-tools
-        uses: actions/cache@v4
-        with:
-          path: ./bin
-          key: build-tools-${{ github.ref_name }}
-
-      - name: Load open-feature-operator image into docker
-        run: |
-          docker load --input ${{ github.workspace }}/open-feature-operator-local.tar
-      - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.9.0
-        with:
-          config: ./test/e2e/kind-cluster.yml
-          cluster_name: open-feature-operator-test
-          node_image: kindest/node:${{ matrix.node_image }}
-      - name: Download KUTTL
-        run: |
-            curl -fL https://github.com/kudobuilder/kuttl/releases/download/${{ env.KUTTL_VERSION }}/kubectl-kuttl_${KUTTL_VERSION#v}_linux_x86_64 -o kubectl-kuttl
-            chmod +x kubectl-kuttl
-            mv kubectl-kuttl /usr/local/bin
-      - name: Load open-feature-operator image into Kind cluster
-        run: |
-          kind load docker-image open-feature-operator-local:${{ github.sha }} --name open-feature-operator-test
-      - name: Run e2e test
-        run: |
-          IMG=open-feature-operator-local:${{ github.sha }} make deploy-operator
-          IMG=open-feature-operator-local:${{ github.sha }} make e2e-test-kuttl
-      - name: Create reports
-        if: failure()
-        working-directory: ./.github/scripts
-        run: ./create-reports.sh
-
-      - name: Upload cluster logs
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: e2e-tests
-          path: .github/scripts/logs
+        kind_tag: [ v1.22.17, v1.23.17, v1.24.15, v1.25.11, v1.26.3, v1.27.3, v1.28.0 ]
+    with:
+      kind_tag: ${{ matrix.kind_tag }}
+    uses: ./.github/workflows/e2e.yml

Makefile

@@ -74,22 +74,25 @@ unit-test: manifests fmt vet generate envtest ## Run tests.
 	cat cover-operator.out cover-apis.out >> cover.out
 	rm cover-operator.out cover-apis.out
 
-## e2e tests require the operator to be deployed in a real cluster
-.PHONY: e2e-test-kuttl
-e2e-test-kuttl:
-	kubectl kuttl test --start-kind=false --config=./kuttl-test.yaml
+############
+# CHAINSAW #
+############
 
-.PHONY: e2e-test-kuttl-local
-e2e-test-kuttl-local:
-	kubectl kuttl test --start-kind=false --config=./kuttl-test-local.yaml
+.PHONY: e2e-test-chainsaw #these tests should run on a real cluster!
+e2e-test-chainsaw:
+	chainsaw test --test-dir ./test/e2e/chainsaw
+
+.PHONY: e2e-test-chainsaw-local #these tests should run on a real cluster!
+e2e-test-chainsaw-local:
+	chainsaw test --test-dir ./test/e2e/chainsaw --config ./.chainsaw-local.yaml
 
 .PHONY: e2e-test-validate-local
 e2e-test-validate-local:
 	docker build . -t open-feature-operator-local:validate
 	kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests
 	kind load docker-image open-feature-operator-local:validate --name e2e-tests
 	IMG=open-feature-operator-local:validate make deploy-operator
-	IMG=open-feature-operator-local:validate make e2e-test-kuttl
+	IMG=open-feature-operator-local:validate make e2e-test-chainsaw
 	kind delete cluster --name e2e-tests
 
 .PHONY: lint

kuttl-test-local.yaml

@@ -87,7 +87,9 @@ spec:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: flagd-query-test
+  name: (join('-', [$namespace, 'job1']))
+  labels:
+    kind: e2e-test-job
 spec:
   backoffLimit: 10
   template:

kuttl-test.yaml

@@ -90,7 +90,9 @@ spec:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: flagd-query-test
+  name: (join('-', [$namespace, 'job1']))
+  labels:
+    kind: flagd-disabled
 spec:
   backoffLimit: 5
   template:

test/e2e/kuttl/assets/manifests.yaml test/e2e/chainsaw/assets/manifests.yaml

@@ -0,0 +1,23 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: flagd-disabled
+spec:
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: 00-install.yaml
+        - wait:
+            timeout: 5m
+            resource: job
+            selector: kind=flagd-disabled
+            for:
+              condition:
+                name: complete
+    - name: step-01
+      try:
+        - assert:
+            file: 00-assert.yaml

test/e2e/kuttl/flagd-disabled/01-assert.yaml test/e2e/chainsaw/flagd-disabled/00-assert.yaml

@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: flagd-with-ingress-custom-paths
+spec:
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: 00-install.yaml
+        - assert:
+            file: 00-assert.yaml

test/e2e/kuttl/flagd-disabled/00-install.yaml test/e2e/chainsaw/flagd-disabled/00-install.yaml

@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: flagd-with-ingress-default-paths
+spec:
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: 00-install.yaml
+        - assert:
+            file: 00-assert.yaml

test/e2e/chainsaw/flagd-disabled/chainsaw-test.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: fsconfig-file-sync
+spec:
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: ../assets/manifests.yaml
+    - name: step-01
+      try:
+        - apply:
+            file: 00-install.yaml
+        - wait:
+            timeout: 5m
+            resource: job
+            selector: kind=e2e-test-job
+            for:
+              condition:
+                name: complete
+    - name: step-02
+      try:
+        - assert:
+            file: 00-assert.yaml

test/e2e/kuttl/flagd-with-ingress-custom-paths/00-assert.yaml test/e2e/chainsaw/flagd-with-ingress-custom-paths/00-assert.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: fsconfig-flagd-proxy-sync
+spec:
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: ../assets/manifests.yaml
+    - name: step-01
+      try:
+        - apply:
+            file: 00-install.yaml
+        - wait:
+            timeout: 5m
+            resource: job
+            selector: kind=e2e-test-job
+            for:
+              condition:
+                name: complete
+    - name: step-02
+      try:
+        - assert:
+            file: 00-assert.yaml

test/e2e/kuttl/flagd-with-ingress-custom-paths/00-install.yaml test/e2e/chainsaw/flagd-with-ingress-custom-paths/00-install.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: fsconfig-k8s-sync
+spec:
+  steps:
+    - name: step-00
+      try:
+        - apply:
+            file: ../assets/manifests.yaml
+    - name: step-01
+      try:
+        - apply:
+            file: 00-install.yaml
+        - wait:
+            timeout: 5m
+            resource: job
+            selector: kind=e2e-test-job
+            for:
+              condition:
+                name: complete
+    - name: step-02
+      try:
+        - assert:
+            file: 00-assert.yaml