diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..3d4c7ff7 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +// For format details, see https://containers.dev +{ + "name": "container-image-csi-driver dev container", + "image": "ghcr.io/warm-metal/container-image-csi-driver/devcontainer:latest", + + // Setup the go environment and mount into the dev container at the expected location + "workspaceFolder": "/go/src/github.com/warm-metal/container-image-csi-driver", + "workspaceMount": "source=${localWorkspaceFolder},target=/go/src/github.com/warm-metal/container-image-csi-driver,type=bind,consistency=cached", + + // Copy over welcome message + "postCreateCommand": "sudo cp .devcontainer/welcome.txt /usr/local/etc/vscode-dev-containers/first-run-notice.txt" +} diff --git a/.devcontainer/welcome.txt b/.devcontainer/welcome.txt new file mode 100644 index 00000000..18b9c34a --- /dev/null +++ b/.devcontainer/welcome.txt @@ -0,0 +1,7 @@ +👋 Welcome to your development environment for Container Image CSI Driver! + +This debian dev container image contains all tools needed for development. + +Get started right away! 🚀 + +Check out the guide in docs/running-locally.md diff --git a/.github/.devcontainer/Dockerfile b/.github/.devcontainer/Dockerfile new file mode 100644 index 00000000..88cae803 --- /dev/null +++ b/.github/.devcontainer/Dockerfile @@ -0,0 +1 @@ +FROM mcr.microsoft.com/devcontainers/base:ubuntu diff --git a/.github/.devcontainer/devcontainer.json b/.github/.devcontainer/devcontainer.json new file mode 100644 index 00000000..b444b86d --- /dev/null +++ b/.github/.devcontainer/devcontainer.json @@ -0,0 +1,47 @@ +// For format details, see https://containers.dev +{ + "name": "container-image-csi-driver dev container", + // Workaround as mentioned in + // https://github.com/devcontainers/ci/issues/191#issuecomment-1473518609 + + //"image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "build": { + "dockerfile": "./Dockerfile", + "context": "." + }, + + "otherPortsAttributes": { + "onAutoForward": "silent" + }, + + "features": { + "ghcr.io/devcontainers/features/go:1": { + "version": "1.19" + }, + "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": { + "version": "latest", + "helm": "latest", + "minikube": "none" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "dockerDashComposeVersion": "v2" + }, + "ghcr.io/devcontainers-contrib/features/kind:1": {} + }, + + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.defaultProfile.linux": "zsh", + "terminal.integrated.profiles.linux": { + "zsh": { + "path": "zsh" + } + } + }, + "extensions": [ + "golang.go" + ] + } + } +} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..25db42cf --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @warm-metal/csi-maintainers diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..4ef387ba --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "chore: [skip ci]" + include: "scope" diff --git a/.github/workflows/backward-compatibility.yaml b/.github/workflows/backward-compatibility.yaml index b0bf8e16..382f432b 100644 --- a/.github/workflows/backward-compatibility.yaml +++ b/.github/workflows/backward-compatibility.yaml @@ -1,17 +1,17 @@ name: backward-compatibility-5mins on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] workflow_dispatch: jobs: integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Start a kind cluster with containerd - uses: helm/kind-action@v1.4.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: kind-${{ github.run_id }} kubectl_version: "v1.25.2" diff --git a/.github/workflows/containerd.yaml b/.github/workflows/containerd.yaml index 9c0af184..082e8c91 100644 --- a/.github/workflows/containerd.yaml +++ b/.github/workflows/containerd.yaml @@ -1,17 +1,17 @@ name: containerd-11mins on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] workflow_dispatch: jobs: integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Start a kind cluster with containerd - uses: helm/kind-action@v1.4.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: kind-${{ github.run_id }} kubectl_version: "v1.25.2" diff --git a/.github/workflows/cri-o.yaml b/.github/workflows/cri-o.yaml index 5302ea90..ca6a3bcd 100644 --- a/.github/workflows/cri-o.yaml +++ b/.github/workflows/cri-o.yaml @@ -1,21 +1,21 @@ name: cri-o-10mins on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] workflow_dispatch: jobs: integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Start a kind cluster with crio - uses: helm/kind-action@v1.4.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: kind-${{ github.run_id }} kubectl_version: "v1.25.2" - node_image: "docker.io/warmmetal/kindest-node-crio:v1.25.2-csi-driver-image" + node_image: "docker.io/warmmetal/kindest-node-crio:v1.25.2-container-image-csi-driver" - name: Install private registry run: ./hack/ci/setup_private_registry.sh - name: Build image diff --git a/.github/workflows/devcontainer-build-and-push.yaml b/.github/workflows/devcontainer-build-and-push.yaml new file mode 100644 index 00000000..bca90391 --- /dev/null +++ b/.github/workflows/devcontainer-build-and-push.yaml @@ -0,0 +1,55 @@ +name: Dev Container Build and Push Image + +on: + push: + branches: [main] + paths: + - .github/workflows/devcontainer-build-and-push.yaml + - '.github/.devcontainer/**' + pull_request: + paths: + - .github/workflows/devcontainer-build-and-push.yaml + - '.github/.devcontainer/**' + +permissions: + contents: read + packages: write + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to the GitHub Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install updated skopeo + # https://github.com/devcontainers/ci/issues/191#issuecomment-1416384710 + run: | + sudo apt purge buildah golang-github-containers-common podman skopeo + sudo apt autoremove --purge + REPO_URL="https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable" + source /etc/os-release + sudo sh -c "echo 'deb ${REPO_URL}/x${NAME}_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list" + sudo wget -qnv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/x${NAME}_${VERSION_ID}/Release.key -O Release.key + sudo apt-key add Release.key + sudo apt-get update + sudo apt-get install skopeo -y + - name: Pre-build dev container image + uses: devcontainers/ci@v0.3 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }}/devcontainer + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer + platform: linux/amd64,linux/arm64 + push: always diff --git a/.github/workflows/docker-containerd.yaml b/.github/workflows/docker-containerd.yaml index 59fdb5df..4e906a32 100644 --- a/.github/workflows/docker-containerd.yaml +++ b/.github/workflows/docker-containerd.yaml @@ -5,7 +5,7 @@ jobs: integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: opsgang/ga-setup-minikube@v0.1.2 with: minikube-version: '1.20.0' diff --git a/.github/workflows/images-build-and-push.yaml b/.github/workflows/images-build-and-push.yaml new file mode 100644 index 00000000..a6123fcb --- /dev/null +++ b/.github/workflows/images-build-and-push.yaml @@ -0,0 +1,65 @@ +name: Create and publish the container image + +on: + push: + branches: + - '*' + tags: + - 'v*' + +env: + BASE_REPOSITORY: warm-metal/container-image-csi-driver + BASE_DEFAULT_BRANCH: main + BASE_IMAGE_NAME: warmmetal/csi-image + FORK_IMAGE_NAME: ghcr.io/${{ github.repository }} + +permissions: + packages: write + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: github.repository == env.BASE_REPOSITORY + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to the GitHub Container registry + uses: docker/login-action@v3 + if: github.repository != env.BASE_REPOSITORY + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: | + name=${{ env.BASE_IMAGE_NAME }},enable=${{ github.repository == env.BASE_REPOSITORY }} + name=${{ env.FORK_IMAGE_NAME }},enable=${{ github.repository != env.BASE_REPOSITORY }} + tags: | + # set 'latest' tag for default branch + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', env.BASE_DEFAULT_BRANCH) }} + # set image tag based on GitHub release tag + type=semver,pattern={{version}},prefix=v + # set the image tag based on the branch + # https://github.com/docker/metadata-action/issues/247#issuecomment-1511259674 + type=ref,event=branch,enable=${{ github.ref_name != env.BASE_DEFAULT_BRANCH }} + flavor: | + latest=false + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/images-cleanup.yaml b/.github/workflows/images-cleanup.yaml new file mode 100644 index 00000000..a87c41d6 --- /dev/null +++ b/.github/workflows/images-cleanup.yaml @@ -0,0 +1,27 @@ +name: Delete old container images + +on: + schedule: + - cron: "0 0 * * *" # Source: https://crontab.guru/every-day + +permissions: + packages: write + +env: + BASE_REPOSITORY: warm-metal/container-image-csi-driver + +jobs: + clean-ghcr: + name: Delete old unused container images + runs-on: ubuntu-latest + steps: + - uses: actions/delete-package-versions@v5 + if: github.repository != env.BASE_REPOSITORY + with: + owner: ${{ github.repository_owner }} + package-name: ${{ github.event.repository.name }} + package-type: 'container' + min-versions-to-keep: 1 + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + # Ignore any semver or 'latest' + ignore-versions: '^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?|latest$' diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml new file mode 100644 index 00000000..4b84bd49 --- /dev/null +++ b/.github/workflows/metrics.yaml @@ -0,0 +1,37 @@ +name: test-metrics-5m +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: +jobs: + integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Start a kind cluster with containerd + uses: helm/kind-action@v1.8.0 + with: + cluster_name: kind-${{ github.run_id }} + kubectl_version: "v1.25.2" + config: ./hack/ci/containerd-cluster-conf.yaml + - name: Build image + run: ./hack/ci/build.sh + - name: Set image version + run: | + echo "VALUE_FILE=charts/warm-metal-csi-driver/values.yaml" >> "$GITHUB_ENV" + echo "IMAGE_TAG=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + echo "HELM_NAME=wm-csi-integration-tests" >> "$GITHUB_ENV" + - name: Install the CSI Driver + run: | + trap "kubectl -n kube-system describe po" ERR + helm install ${HELM_NAME} charts/warm-metal-csi-driver -n kube-system \ + -f ${VALUE_FILE} \ + --set csiPlugin.image.tag=${IMAGE_TAG} \ + --wait \ + --debug + - name: Test metrics + run: ./test/integration/test-metrics.sh + - name: Uninstall the CSI Driver + run: helm uninstall -n kube-system ${HELM_NAME} --wait diff --git a/.github/workflows/restart-ds-containerd.yaml b/.github/workflows/restart-ds-containerd.yaml index da9e7469..8cf26202 100644 --- a/.github/workflows/restart-ds-containerd.yaml +++ b/.github/workflows/restart-ds-containerd.yaml @@ -1,17 +1,17 @@ name: restart-ds-containerd-5mins on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] workflow_dispatch: jobs: integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Start a kind cluster with containerd - uses: helm/kind-action@v1.4.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: kind-${{ github.run_id }} kubectl_version: "v1.25.2" diff --git a/.github/workflows/restart-ds-crio.yaml b/.github/workflows/restart-ds-crio.yaml index 89f621d0..855ec852 100644 --- a/.github/workflows/restart-ds-crio.yaml +++ b/.github/workflows/restart-ds-crio.yaml @@ -1,21 +1,21 @@ name: restart-ds-crio-8mins on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] workflow_dispatch: jobs: integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Start a kind cluster with crio - uses: helm/kind-action@v1.4.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: kind-${{ github.run_id }} kubectl_version: "v1.25.2" - node_image: "docker.io/warmmetal/kindest-node-crio:v1.25.2-csi-driver-image" + node_image: "docker.io/warmmetal/kindest-node-crio:v1.25.2-container-image-csi-driver" - name: Build image run: ./hack/ci/build.sh - name: Set image version diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 00000000..c9c2b0dd --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,46 @@ +name: Close stale issues and PRs + +on: + push: + paths: + - .github/workflows/stale.yml + branches-ignore: + - dependabot/** + - releases/** + schedule: + # Once every day at midnight UTC. Source: https://crontab.guru/every-day + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + # Setting this variable to 'true' will cause the + # action to run in a dry-run mode + debug-only: ${{ vars.DEBUG_ONLY == 'true' }} + stale-issue-message: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed in 7 days if no further activity occurs. + close-issue-message: > + Closing this issue after a prolonged period of inactivity. If this + issue is still present in the latest release, please feel free to + create a new issue with up-to-date information. + stale-pr-message: > + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed in 7 days if no further activity occurs. + close-pr-message: > + Closing this PR after a prolonged period of inactivity. Please create a new PR + if the changes of the PR are still relevant. + exempt-all-milestones: true + exempt-issue-labels: 'awaiting-inputs,work-in-progress,help-wanted' + exempt-pr-labels: 'awaiting-inputs,work-in-progress,help-wanted' + days-before-issue-stale: 90 + days-before-issue-close: 15 + days-before-pr-stale: 30 + days-before-pr-close: 7 diff --git a/Dockerfile b/Dockerfile index a91ae276..d4ea9ffc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM docker.io/library/golang:1.20.4-alpine3.17 as builder RUN apk add --no-cache btrfs-progs-dev lvm2-dev make build-base -WORKDIR /go/src/csi-driver-image +WORKDIR /go/src/container-image-csi-driver COPY go.mod go.sum ./ RUN go mod download COPY cmd ./cmd @@ -10,10 +10,10 @@ RUN make build RUN make install-util FROM scratch as install-util -COPY --from=builder /go/src/csi-driver-image/_output/warm-metal-csi-image-install / +COPY --from=builder /go/src/container-image-csi-driver/_output/warm-metal-csi-image-install / FROM alpine:3.17 RUN apk add --no-cache btrfs-progs-dev lvm2-dev WORKDIR / -COPY --from=builder /go/src/csi-driver-image/_output/csi-image-plugin /usr/bin/ +COPY --from=builder /go/src/container-image-csi-driver/_output/csi-image-plugin /usr/bin/ ENTRYPOINT ["csi-image-plugin"] diff --git a/Makefile b/Makefile index 2a371178..c10cca6d 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ -VERSION ?= v0.7.0 +VERSION ?= v1.1.0 IMAGE_BUILDER ?= docker IMAGE_BUILD_CMD ?= buildx REGISTRY ?= docker.io/warmmetal +PLATFORM ?= linux/amd64 export IMG = $(REGISTRY)/csi-image:$(VERSION) @@ -16,15 +17,24 @@ build: .PHONY: sanity sanity: - $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build -t local.test/csi-driver-image-test:sanity test/sanity + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build -t local.test/container-image-csi-driver-test:sanity test/sanity kubectl delete --ignore-not-found -f test/sanity/manifest.yaml kubectl apply --wait -f test/sanity/manifest.yaml - kubectl -n cliapp-system wait --for=condition=complete job/csi-driver-image-sanity-test + kubectl -n cliapp-system wait --for=condition=complete job/container-image-csi-driver-sanity-test .PHONY: e2e e2e: cd ./test/e2e && KUBECONFIG=~/.kube/config go run . +# to run unit tests +# PHONY signifies the make recipe is not a file +# more info: https://stackoverflow.com/questions/2145590/what-is-the-purpose-of-phony-in-a-makefile +.PHONY: unit-tests +unit-tests: + # -count=1 forces re-test everytime (instead of caching the results) + # more info: https://stackoverflow.com/questions/48882691/force-retesting-or-disable-test-caching + go test -count=1 ./cmd/plugin + .PHONY: integration integration: ./test/integration/test-backward-compatability.sh @@ -35,17 +45,18 @@ integration: .PHONY: image image: - $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build -t $(REGISTRY)/csi-image:$(VERSION) --push . + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --platform=$(PLATFORM) -t $(REGISTRY)/csi-image:$(VERSION) --push . .PHONY: local local: - $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build -t $(REGISTRY)/csi-image:$(VERSION) --load . + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --platform=$(PLATFORM) -t $(REGISTRY)/csi-image:$(VERSION) --load . .PHONY: test-deps test-deps: - $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --push -t $(REGISTRY)/csi-image-test:stat-fs -f csi-image-test:stat-fs.dockerfile hack/integration-test-image - $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --push -t $(REGISTRY)/csi-image-test:check-fs -f csi-image-test:check-fs.dockerfile hack/integration-test-image - $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --push -t $(REGISTRY)/csi-image-test:write-check -f csi-image-test:write-check.dockerfile hack/integration-test-image + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --platform=$(PLATFORM) --push -t $(REGISTRY)/container-image-csi-driver-test:simple-fs -f hack/integration-test-image/container-image-csi-driver-test:simple-fs.dockerfile hack/integration-test-image + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --platform=$(PLATFORM) --push -t $(REGISTRY)/container-image-csi-driver-test:stat-fs -f hack/integration-test-image/container-image-csi-driver-test:stat-fs.dockerfile hack/integration-test-image + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --platform=$(PLATFORM) --push -t $(REGISTRY)/container-image-csi-driver-test:check-fs -f hack/integration-test-image/container-image-csi-driver-test:check-fs.dockerfile hack/integration-test-image + $(IMAGE_BUILDER) $(IMAGE_BUILD_CMD) build --platform=$(PLATFORM) --push -t $(REGISTRY)/container-image-csi-driver-test:write-check -f hack/integration-test-image/container-image-csi-driver-test:write-check.dockerfile hack/integration-test-image .PHONY: install-util install-util: diff --git a/README.md b/README.md index 372b42da..8624b892 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,33 @@ # -[![containerd](https://github.com/warm-metal/csi-driver-image/actions/workflows/containerd.yaml/badge.svg)](https://github.com/warm-metal/csi-driver-image/actions/workflows/containerd.yaml) -[![docker-containerd](https://github.com/warm-metal/csi-driver-image/actions/workflows/docker-containerd.yaml/badge.svg)](https://github.com/warm-metal/csi-driver-image/actions/workflows/docker-containerd.yaml) -[![cri-o](https://github.com/warm-metal/csi-driver-image/actions/workflows/cri-o.yaml/badge.svg)](https://github.com/warm-metal/csi-driver-image/actions/workflows/cri-o.yaml) +[![containerd](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/containerd.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/containerd.yaml) +[![docker-containerd](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/docker-containerd.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/docker-containerd.yaml) +[![cri-o](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/cri-o.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/cri-o.yaml) ![Docker Pulls](https://img.shields.io/docker/pulls/warmmetal/csi-image?color=brightgreen&logo=docker&logoColor=lightgrey&labelColor=black) +# :construction_worker_man: :wrench: :construction: RENAMING THE REPOSITORY :construction: :wrench: :construction_worker_man: + +We are currently in the process of [changing the repository name](https://github.com/warm-metal/container-image-csi-driver/issues/105). This alteration may potentially introduce issues during Continuous Integration (CI) runs or while building packages locally. If you encounter any problems, we encourage you to promptly create an issue so that we can assist you in resolving them. + +### Note for Forked Repositories: +If you have forked this repository before January 21, 2024, we kindly request that you follow the steps outlined in the [GitHub documentation](https://docs.github.com/en/repositories/creating-and-managing-repositories/renaming-a-repository) to update your remote. This ensures that your fork remains synchronized with the latest changes and avoids any disruption to your workflow. + +Also the default branch has been updated to `main` from `master`. Please run below commands for updating your local setup. +``` +git branch -m master main +git fetch origin +git branch -u main main +git remote set-head origin -a +``` + +We appreciate your cooperation and understanding as we work to improve our repository. + +# container-image-csi-driver (previously csi-driver-image) + This is a CSI driver for mounting images as PVs or ephemeral volumes. -It pulls images via CRI and shares the image store with the container runtime, +It pulls images via CRI and shares the image store with the container runtime, then mounts images via the snapshot/storage service of the runtime. **Read-Only** volumes of the same image share the same snapshot. **Read-Write** volumes keep their own snapshot and changes until pod deletion. @@ -26,13 +45,27 @@ then mounts images via the snapshot/storage service of the runtime. * [Pre-provisioned PV](#pre-provisioned-pv) * [Private Image](#private-image) +## Compatibility matrix +Tested changes on below mentioned versions - + +| warm-metal | k8s version | containerd | crio | +|------------|-------------|------------|---------| +| 0.6.x | v1.25 | 1.6.8 | v1.20.9 | +| 0.7.x | v1.25 | 1.6.8 | v1.20.9 | +| 0.8.x | v1.25 | 1.6.8 | v1.20.9 | +| 1.0.x | v1.25 | 1.6.8 | v1.25.2 | + +#### References: +* containerd [releases](https://containerd.io/releases/#kubernetes-support) +* cri-o [releases](https://github.com/cri-o/cri-o/releases) + ## Installation The driver requires to mount various host paths for different container runtimes. So, I build a binary utility, `warm-metal-csi-image-install`, to reduce the installation complexity. It supports kubernetes, microk8s and k3s clusters with container runtime **cri-o**, **containerd** or **docker**. Users can run this utility on any nodes in their clusters to generate proper manifests to install the driver. -The download link is available on the [release page](https://github.com/warm-metal/csi-driver-image/releases). +The download link is available on the [release page](https://github.com/warm-metal/container-image-csi-driver/releases). ```shell script # To print manifests @@ -51,13 +84,13 @@ warm-metal-csi-image-install --pull-image-secret-for-daemonset=foo warm-metal-csi-image-install --pull-image-secret-for-daemonset=foo --enable-daemon-image-credential-cache=false ``` -You can found some installation manifests as samples in [examples](https://github.com/warm-metal/csi-driver-image/tree/master/sample). +You can found some installation manifests as samples in [examples](https://github.com/warm-metal/container-image-csi-driver/tree/master/sample). #### Notice for docker Until Docker migrates its [image and snapshot store](https://github.com/moby/moby/issues/38043) to containerd, I recommend you use containerd instead. Otherwise, the driver can't use images managed by Docker daemon. -If your container runtime can't be migrated, you can enable the CRI plugin by clearing +If your container runtime can't be migrated, you can enable the CRI plugin by clearing the containerd config file `/etc/containerd/config.toml`, then restarting the containerd. @@ -82,7 +115,7 @@ spec: spec: containers: - name: ephemeral-volume - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -97,7 +130,7 @@ spec: # nodePublishSecretRef: # name: "ImagePullSecret name in the same namespace" volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" # # set pullAlways if you want to ignore local images # pullAlways: "true" backoffLimit: 0 @@ -110,7 +143,7 @@ For pre-provisioned PVs, `volumeHandle` instead of the attribute **image**, spec apiVersion: v1 kind: PersistentVolume metadata: - name: pv-test-csi-image-test-simple-fs + name: pv-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech capacity: @@ -120,7 +153,7 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" # nodePublishSecretRef: # name: "name of the ImagePullSecret" # namespace: "namespace of the secret" @@ -129,14 +162,14 @@ spec: # pullAlways: "true" ``` -See all [examples](https://github.com/warm-metal/csi-driver-image/tree/master/sample). +See all [examples](https://github.com/warm-metal/container-image-csi-driver/tree/master/sample). #### Private Image -There are several ways to configure credentials for private image pulling. +There are several ways to configure credentials for private image pulling. If your clusters are in cloud, the credential provider are enabled automatically. -If your cloud provider provides a credential provider plugin instead, you can enable it by adding +If your cloud provider provides a credential provider plugin instead, you can enable it by adding both `--image-credential-provider-config` and `--image-credential-provider-bin-dir` flags to the driver. See [credential provider plugin](https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider/). @@ -153,7 +186,7 @@ then restart the driver daemon pod. Users can run `warm-metal-csi-image-install` warm-metal-csi-image-install --pull-image-secret-for-daemonset=foo --pull-image-secret-for-daemonset=bar ``` -If the secret works only for particular workloads, you can set via the `nodePublishSecretRef` attribute of ephemeral volumes. +If the secret works only for particular workloads, you can set via the `nodePublishSecretRef` attribute of ephemeral volumes. See the above sample manifest, and notice that secrets and workloads must in the same namespace. (Since version v0.5.1, pulling private images using the ImagePullSecrets which attached to workload service accounts is no longer supported for security reasons.) @@ -163,8 +196,18 @@ You can also set the secret to a PV, then share the PV with multiple workloads. ### Sanity test -See [test/sanity](https://github.com/warm-metal/csi-driver-image/tree/master/test/sanity). +See [test/sanity](https://github.com/warm-metal/container-image-csi-driver/tree/master/test/sanity). ### E2E test -See [test/e2e](https://github.com/warm-metal/csi-driver-image/tree/master/test/e2e). +See [test/e2e](https://github.com/warm-metal/container-image-csi-driver/tree/master/test/e2e). + +## Note on logging image size +Image sizes are logged after they finish pulling. We've noticed that for smaller images, usually under 1KiB, containerd may report an incorrect image size. An issue has been raised in the containerd github repository: https://github.com/containerd/containerd/issues/9641. + +## Community meetings +We conduct online meetings every 1st, 3rd, and 5th week of the month on Thursdays at 15:30 UTC. + +Feel free to join us through [this Zoom link](https://acquia.zoom.us/j/94346685583) and refer to the [Google document](https://docs.google.com/document/d/1nDiRtj85ZpWMH57joUmbtGG3aLkfeqFt6OWdkf8_Aaw/edit?usp=sharing) for Minutes of Meetings (MoMs). + +Note: If you are unable to attend the meeting but still interested, you can initiate a discussion under Discussions/Queries/Suggestions in the aforementioned Google document. diff --git a/charts/warm-metal-csi-driver/Chart.yaml b/charts/warm-metal-csi-driver/Chart.yaml index a5fbff95..c9418412 100644 --- a/charts/warm-metal-csi-driver/Chart.yaml +++ b/charts/warm-metal-csi-driver/Chart.yaml @@ -15,9 +15,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.7.0 +version: 1.1.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: v0.7.0 +appVersion: v1.1.0 diff --git a/charts/warm-metal-csi-driver/templates/nodeplugin.yaml b/charts/warm-metal-csi-driver/templates/nodeplugin.yaml index d7fdc37d..a65aaddd 100644 --- a/charts/warm-metal-csi-driver/templates/nodeplugin.yaml +++ b/charts/warm-metal-csi-driver/templates/nodeplugin.yaml @@ -31,13 +31,6 @@ spec: fieldPath: spec.nodeName image: "{{ .Values.csiNodeDriverRegistrar.image.repository }}:{{ .Values.csiNodeDriverRegistrar.image.tag }}" imagePullPolicy: {{ .Values.csiNodeDriverRegistrar.image.pullPolicy }} - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/csi-image.warm-metal.tech /registration/csi-image.warm-metal.tech-reg.sock {{- with .Values.csiNodeDriverRegistrar.resources }} resources: {{- toYaml . | nindent 12 }} @@ -70,6 +63,9 @@ spec: {{- if .Values.enableDaemonImageCredentialCache }} - --enable-daemon-image-credential-cache {{- end }} + {{- if .Values.enableAsyncPullMount }} + - --async-pull-mount=true + {{- end }} - "-v={{ .Values.logLevel }}" - "--mode=node" env: @@ -92,6 +88,9 @@ spec: - containerPort: 9809 name: metrics protocol: TCP + - containerPort: 8080 + name: metrics2 + protocol: TCP livenessProbe: {{- toYaml .Values.csiPlugin.livenessProbe | nindent 12}} securityContext: @@ -116,7 +115,7 @@ spec: - mountPath: {{ .Values.crioMountProgram }} name: crio-mount-program {{- end }} - hostNetwork: true + hostNetwork: {{.Values.csiPlugin.hostNetwork}} serviceAccountName: {{ include "warm-metal-csi-driver.fullname" . }}-nodeplugin volumes: - hostPath: diff --git a/charts/warm-metal-csi-driver/templates/podmonitor.yaml b/charts/warm-metal-csi-driver/templates/podmonitor.yaml index 9a29a464..15081591 100644 --- a/charts/warm-metal-csi-driver/templates/podmonitor.yaml +++ b/charts/warm-metal-csi-driver/templates/podmonitor.yaml @@ -15,6 +15,15 @@ spec: {{- if .Values.podMonitor.timeout }} scrapeTimeout: {{ .Values.podMonitor.timeout }} {{- end }} + - path: /metrics + port: metrics2 + scheme: http + {{- if .Values.podMonitor.interval }} + interval: {{ .Values.podMonitor.interval }} + {{- end }} + {{- if .Values.podMonitor.timeout }} + scrapeTimeout: {{ .Values.podMonitor.timeout }} + {{- end }} jobLabel: {{ include "warm-metal-csi-driver.fullname" . }} namespaceSelector: matchNames: diff --git a/charts/warm-metal-csi-driver/values.yaml b/charts/warm-metal-csi-driver/values.yaml index fc6678f8..cf577ff6 100644 --- a/charts/warm-metal-csi-driver/values.yaml +++ b/charts/warm-metal-csi-driver/values.yaml @@ -5,9 +5,11 @@ kubeletRoot: /var/lib/kubelet snapshotRoot: /var/lib/containerd/io.containerd.snapshotter.v1.overlayfs logLevel: 4 enableDaemonImageCredentialCache: +enableAsyncPullMount: false pullImageSecretForDaemonset: csiPlugin: + hostNetwork: false resources: {} image: tag: "" diff --git a/cmd/install/cri.go b/cmd/install/cri.go index 6dc6a660..56b4013f 100644 --- a/cmd/install/cri.go +++ b/cmd/install/cri.go @@ -15,7 +15,7 @@ import ( "github.com/BurntSushi/toml" "google.golang.org/grpc" corev1 "k8s.io/api/core/v1" - criapis "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + criapis "k8s.io/cri-api/pkg/apis/runtime/v1" ) var typeDir = corev1.HostPathDirectory diff --git a/cmd/install/main.go b/cmd/install/main.go index 8b66de87..2c82a624 100644 --- a/cmd/install/main.go +++ b/cmd/install/main.go @@ -278,7 +278,7 @@ spec: labels: app: csi-image-warm-metal spec: - hostNetwork: true + hostNetwork: false serviceAccountName: csi-image-warm-metal containers: - name: node-driver-registrar diff --git a/cmd/plugin/controller_server.go b/cmd/plugin/controller_server.go index d72b9793..e8e52dc9 100644 --- a/cmd/plugin/controller_server.go +++ b/cmd/plugin/controller_server.go @@ -5,7 +5,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/pkg/errors" - "github.com/warm-metal/csi-driver-image/pkg/watcher" + "github.com/warm-metal/container-image-csi-driver/pkg/watcher" csicommon "github.com/warm-metal/csi-drivers/pkg/csi-common" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/cmd/plugin/main.go b/cmd/plugin/main.go index 550b889f..4e2994f2 100644 --- a/cmd/plugin/main.go +++ b/cmd/plugin/main.go @@ -5,19 +5,19 @@ import ( goflag "flag" "fmt" "net/url" + "time" "github.com/container-storage-interface/spec/lib/go/csi" flag "github.com/spf13/pflag" - "github.com/warm-metal/csi-driver-image/pkg/backend" - "github.com/warm-metal/csi-driver-image/pkg/backend/containerd" - "github.com/warm-metal/csi-driver-image/pkg/backend/crio" - "github.com/warm-metal/csi-driver-image/pkg/cri" - "github.com/warm-metal/csi-driver-image/pkg/secret" - "github.com/warm-metal/csi-driver-image/pkg/watcher" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/backend/containerd" + "github.com/warm-metal/container-image-csi-driver/pkg/backend/crio" + "github.com/warm-metal/container-image-csi-driver/pkg/cri" + "github.com/warm-metal/container-image-csi-driver/pkg/metrics" + "github.com/warm-metal/container-image-csi-driver/pkg/secret" + "github.com/warm-metal/container-image-csi-driver/pkg/watcher" csicommon "github.com/warm-metal/csi-drivers/pkg/csi-common" "k8s.io/klog/v2" - - "time" ) const ( @@ -98,7 +98,7 @@ func main() { *runtimeAddr = addr.String() } - var mounter *backend.SnapshotMounter + var mounter backend.Mounter if len(*runtimeAddr) > 0 { addr, err := url.Parse(*runtimeAddr) if err != nil { @@ -145,5 +145,6 @@ func main() { ) } + metrics.StartMetricsServer(metrics.RegisterMetrics()) server.Wait() } diff --git a/cmd/plugin/node_server.go b/cmd/plugin/node_server.go index 108de79c..109eb5c3 100644 --- a/cmd/plugin/node_server.go +++ b/cmd/plugin/node_server.go @@ -7,15 +7,17 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/containerd/containerd/reference/docker" - "github.com/warm-metal/csi-driver-image/pkg/backend" - "github.com/warm-metal/csi-driver-image/pkg/mountexecutor" - "github.com/warm-metal/csi-driver-image/pkg/mountstatus" - "github.com/warm-metal/csi-driver-image/pkg/pullexecutor" - "github.com/warm-metal/csi-driver-image/pkg/secret" + "github.com/google/uuid" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/metrics" + "github.com/warm-metal/container-image-csi-driver/pkg/mountexecutor" + "github.com/warm-metal/container-image-csi-driver/pkg/mountstatus" + "github.com/warm-metal/container-image-csi-driver/pkg/pullexecutor" + "github.com/warm-metal/container-image-csi-driver/pkg/secret" csicommon "github.com/warm-metal/csi-drivers/pkg/csi-common" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - cri "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + cri "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" k8smount "k8s.io/mount-utils" ) @@ -29,7 +31,7 @@ const ( type ImagePullStatus int -func NewNodeServer(driver *csicommon.CSIDriver, mounter *backend.SnapshotMounter, imageSvc cri.ImageServiceClient, secretStore secret.Store, asyncImagePullMount bool) *NodeServer { +func NewNodeServer(driver *csicommon.CSIDriver, mounter backend.Mounter, imageSvc cri.ImageServiceClient, secretStore secret.Store, asyncImagePullMount bool) *NodeServer { return &NodeServer{ DefaultNodeServer: csicommon.NewDefaultNodeServer(driver), mounter: mounter, @@ -50,7 +52,7 @@ func NewNodeServer(driver *csicommon.CSIDriver, mounter *backend.SnapshotMounter type NodeServer struct { *csicommon.DefaultNodeServer - mounter *backend.SnapshotMounter + mounter backend.Mounter secretStore secret.Store asyncImagePullMount bool mountExecutor *mountexecutor.MountExecutor @@ -58,7 +60,8 @@ type NodeServer struct { } func (n NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (resp *csi.NodePublishVolumeResponse, err error) { - klog.Infof("mount request: %s", req.String()) + valuesLogger := klog.LoggerWithValues(klog.NewKlogr(), "pod-name", req.VolumeContext["pod-name"], "namespace", req.VolumeContext["namespace"], "uid", req.VolumeContext["uid"], "request-id", uuid.NewString()) + valuesLogger.Info("Incoming NodePublishVolume request", "request string", req.String()) if len(req.VolumeId) == 0 { err = status.Error(codes.InvalidArgument, "VolumeId is missing") return @@ -98,7 +101,7 @@ func (n NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishV return } - if err = os.MkdirAll(req.TargetPath, 0755); err != nil { + if err = os.MkdirAll(req.TargetPath, 0o755); err != nil { err = status.Error(codes.Internal, err.Error()) return } @@ -133,6 +136,7 @@ func (n NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishV PullAlways: pullAlways, Image: image, PullSecrets: req.Secrets, + Logger: valuesLogger, } if e := n.pullExecutor.StartPulling(po); e != nil { @@ -141,7 +145,7 @@ func (n NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishV } if e := n.pullExecutor.WaitForPull(po); e != nil { - err = status.Errorf(codes.DeadlineExceeded, err.Error()) + err = status.Errorf(codes.DeadlineExceeded, e.Error()) return } @@ -156,15 +160,16 @@ func (n NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishV TargetPath: req.TargetPath, VolumeCapability: req.VolumeCapability, ReadOnly: req.Readonly, + Logger: valuesLogger, } - if err = n.mountExecutor.StartMounting(o); err != nil { - err = status.Error(codes.Internal, err.Error()) + if e := n.mountExecutor.StartMounting(o); e != nil { + err = status.Error(codes.Internal, e.Error()) return } if e := n.mountExecutor.WaitForMount(o); e != nil { - err = status.Errorf(codes.DeadlineExceeded, err.Error()) + err = status.Errorf(codes.DeadlineExceeded, e.Error()) return } @@ -189,6 +194,8 @@ func (n NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpubl } if err = n.mounter.Unmount(ctx, req.VolumeId, backend.MountTarget(req.TargetPath)); err != nil { + // TODO(vadasambar): move this to mountexecutor once mountexecutor has `StartUnmounting` function + metrics.OperationErrorsCount.WithLabelValues("StartUnmounting").Inc() err = status.Error(codes.Internal, err.Error()) return } diff --git a/cmd/plugin/node_server_test.go b/cmd/plugin/node_server_test.go index 12ae09d6..5db8ac49 100644 --- a/cmd/plugin/node_server_test.go +++ b/cmd/plugin/node_server_test.go @@ -3,7 +3,9 @@ package main import ( "context" "fmt" + "io" "net" + "net/http" "net/url" "os" "strings" @@ -12,11 +14,12 @@ import ( "time" "github.com/container-storage-interface/spec/lib/go/csi" - csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" - "github.com/warm-metal/csi-driver-image/pkg/backend" - "github.com/warm-metal/csi-driver-image/pkg/backend/containerd" - "github.com/warm-metal/csi-driver-image/pkg/cri" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/backend/containerd" + "github.com/warm-metal/container-image-csi-driver/pkg/cri" + "github.com/warm-metal/container-image-csi-driver/pkg/metrics" + "github.com/warm-metal/container-image-csi-driver/pkg/test/utils" csicommon "github.com/warm-metal/csi-drivers/pkg/csi-common" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/util/wait" @@ -24,18 +27,16 @@ import ( "k8s.io/kubernetes/pkg/credentialprovider" ) -// Check test/integration/node-server/README.md for how to run this test correctly func TestNodePublishVolumeAsync(t *testing.T) { - socketAddr := "unix:///run/containerd/containerd.sock" - addr, err := url.Parse(socketAddr) - assert.NoError(t, err) - - criClient, err := cri.NewRemoteImageService(socketAddr, time.Minute) - assert.NoError(t, err) - assert.NotNil(t, criClient) + criClient := &utils.MockImageServiceClient{ + PulledImages: make(map[string]bool), + ImagePullTime: time.Second * 5, + } - mounter := containerd.NewMounter(addr.Path) - assert.NotNil(t, mounter) + mounter := &utils.MockMounter{ + ImageSvcClient: *criClient, + Mounted: make(map[string]bool), + } driver := csicommon.NewCSIDriver(driverName, driverVersion, "fake-node") assert.NotNil(t, driver) @@ -43,7 +44,7 @@ func TestNodePublishVolumeAsync(t *testing.T) { asyncImagePulls := true ns := NewNodeServer(driver, mounter, criClient, &testSecretStore{}, asyncImagePulls) - // based on kubelet's csi mounter pluginc ode + // based on kubelet's csi mounter plugin code // check https://github.com/kubernetes/kubernetes/blob/b06a31b87235784bad2858be62115049b6eb6bcd/pkg/volume/csi/csi_mounter.go#L111-L112 timeout := 100 * time.Millisecond @@ -55,6 +56,10 @@ func TestNodePublishVolumeAsync(t *testing.T) { VolumeContext: map[string]string{ // so that the test would always attempt to pull an image ctxKeyPullAlways: "true", + // to see improved logs + "pod-name": "test-pod", + "namespace": "test-namespace", + "uid": "test-uid", }, VolumeCapability: &csi.VolumeCapability{ AccessMode: &csi.VolumeCapability_AccessMode{ @@ -65,21 +70,23 @@ func TestNodePublishVolumeAsync(t *testing.T) { server := csicommon.NewNonBlockingGRPCServer() - addr, err = url.Parse(*endpoint) - assert.NoError(t, err) - - os.Remove("/csi/csi.sock") + endpoint := "unix:///tmp/csi.sock" // automatically deleted when the server is stopped - f, err := os.Create("/csi/csi.sock") + f, err := os.Create("/tmp/csi.sock") assert.NoError(t, err) assert.NotNil(t, f) + defer os.Remove("/tmp/csi.sock") + + addr, err := url.Parse("unix:///tmp/csi.sock") + assert.NoError(t, err) + var wg sync.WaitGroup wg.Add(1) go func() { - server.Start(*endpoint, + server.Start(endpoint, nil, nil, ns) @@ -113,7 +120,7 @@ func TestNodePublishVolumeAsync(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, conn) - nodeClient := csipbv1.NewNodeClient(conn) + nodeClient := csi.NewNodeClient(conn) assert.NotNil(t, nodeClient) condFn := func() (done bool, err error) { @@ -149,16 +156,14 @@ func TestNodePublishVolumeAsync(t *testing.T) { // Check test/integration/node-server/README.md for how to run this test correctly func TestNodePublishVolumeSync(t *testing.T) { - socketAddr := "unix:///run/containerd/containerd.sock" - addr, err := url.Parse(socketAddr) - assert.NoError(t, err) - - criClient, err := cri.NewRemoteImageService(socketAddr, time.Minute) - assert.NoError(t, err) - assert.NotNil(t, criClient) - - mounter := containerd.NewMounter(addr.Path) - assert.NotNil(t, mounter) + criClient := &utils.MockImageServiceClient{ + PulledImages: make(map[string]bool), + ImagePullTime: time.Second * 5, + } + mounter := &utils.MockMounter{ + ImageSvcClient: *criClient, + Mounted: make(map[string]bool), + } driver := csicommon.NewCSIDriver(driverName, driverVersion, "fake-node") assert.NotNil(t, driver) @@ -166,7 +171,7 @@ func TestNodePublishVolumeSync(t *testing.T) { asyncImagePulls := false ns := NewNodeServer(driver, mounter, criClient, &testSecretStore{}, asyncImagePulls) - // based on kubelet's csi mounter pluginc ode + // based on kubelet's csi mounter plugin code // check https://github.com/kubernetes/kubernetes/blob/b06a31b87235784bad2858be62115049b6eb6bcd/pkg/volume/csi/csi_mounter.go#L111-L112 timeout := 100 * time.Millisecond @@ -178,6 +183,10 @@ func TestNodePublishVolumeSync(t *testing.T) { VolumeContext: map[string]string{ // so that the test would always attempt to pull an image ctxKeyPullAlways: "true", + // to see improved logs + "pod-name": "test-pod", + "namespace": "test-namespace", + "uid": "test-uid", }, VolumeCapability: &csi.VolumeCapability{ AccessMode: &csi.VolumeCapability_AccessMode{ @@ -188,21 +197,23 @@ func TestNodePublishVolumeSync(t *testing.T) { server := csicommon.NewNonBlockingGRPCServer() - addr, err = url.Parse(*endpoint) - assert.NoError(t, err) - - os.Remove("/csi/csi.sock") + endpoint := "unix:///tmp/csi.sock" // automatically deleted when the server is stopped - f, err := os.Create("/csi/csi.sock") + f, err := os.Create("/tmp/csi.sock") assert.NoError(t, err) assert.NotNil(t, f) + defer os.Remove("/tmp/csi/csi.sock") + + addr, err := url.Parse("unix:///tmp/csi.sock") + assert.NoError(t, err) + var wg sync.WaitGroup wg.Add(1) go func() { - server.Start(*endpoint, + server.Start(endpoint, nil, nil, ns) @@ -236,7 +247,7 @@ func TestNodePublishVolumeSync(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, conn) - nodeClient := csipbv1.NewNodeClient(conn) + nodeClient := csi.NewNodeClient(conn) assert.NotNil(t, nodeClient) condFn := func() (done bool, err error) { @@ -273,9 +284,175 @@ func TestNodePublishVolumeSync(t *testing.T) { assert.ErrorContains(t, err, "not found") } -type testSecretStore struct { +// Check test/integration/node-server/README.md for how to run this test correctly +func TestMetrics(t *testing.T) { + socketAddr := "unix:///run/containerd/containerd.sock" + addr, err := url.Parse(socketAddr) + assert.NoError(t, err) + + criClient, err := cri.NewRemoteImageService(socketAddr, time.Minute) + assert.NoError(t, err) + assert.NotNil(t, criClient) + + mounter := containerd.NewMounter(addr.Path) + assert.NotNil(t, mounter) + + driver := csicommon.NewCSIDriver(driverName, driverVersion, "fake-node") + assert.NotNil(t, driver) + + asyncImagePulls := true + ns := NewNodeServer(driver, mounter, criClient, &testSecretStore{}, asyncImagePulls) + + // based on kubelet's csi mounter plugin code + // check https://github.com/kubernetes/kubernetes/blob/b06a31b87235784bad2858be62115049b6eb6bcd/pkg/volume/csi/csi_mounter.go#L111-L112 + timeout := 10 * time.Second + + server := csicommon.NewNonBlockingGRPCServer() + + addr, err = url.Parse(*endpoint) + assert.NoError(t, err) + + os.Remove("/csi/csi.sock") + + // automatically deleted when the server is stopped + f, err := os.Create("/csi/csi.sock") + assert.NoError(t, err) + assert.NotNil(t, f) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + metrics.StartMetricsServer(metrics.RegisterMetrics()) + + server.Start(*endpoint, + nil, + nil, + ns) + // wait for the GRPC server to start + wg.Done() + server.Wait() + }() + + // give some time for server to start + time.Sleep(2 * time.Second) + defer func() { + klog.Info("server was stopped") + server.Stop() + }() + + wg.Wait() + var conn *grpc.ClientConn + + conn, err = grpc.Dial( + addr.Path, + grpc.WithInsecure(), + grpc.WithContextDialer(func(ctx context.Context, targetPath string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", targetPath) + }), + ) + + if err != nil { + panic(err) + } + + assert.NoError(t, err) + assert.NotNil(t, conn) + + nodeClient := csi.NewNodeClient(conn) + assert.NotNil(t, nodeClient) + + ctx, cancel := context.WithTimeout(context.Background(), 3*timeout) + defer cancel() + // wrong image id + wrongVolId := "docker.io-doesnt-exist/library/redis-doesnt-exist:latest" + wrongTargetPath := "wrong-test-path" + wrongReq := &csi.NodePublishVolumeRequest{ + VolumeId: wrongVolId, + TargetPath: wrongTargetPath, + VolumeContext: map[string]string{ + // so that the test would always attempt to pull an image + ctxKeyPullAlways: "true", + // to see improved logs + "pod-name": "test-pod", + "namespace": "test-namespace", + "uid": "test-uid", + }, + VolumeCapability: &csi.VolumeCapability{ + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, + }, + }, + } + + r, err := nodeClient.NodePublishVolume(ctx, wrongReq) + assert.Error(t, err) + assert.Nil(t, r) + + volId := "docker.io/library/redis:latest" + targetPath := "test-path" + req := &csi.NodePublishVolumeRequest{ + VolumeId: volId, + TargetPath: targetPath, + VolumeContext: map[string]string{ + // so that the test would always attempt to pull an image + ctxKeyPullAlways: "true", + // to see improved logs + "pod-name": "test-pod", + "namespace": "test-namespace", + "uid": "test-uid", + }, + VolumeCapability: &csi.VolumeCapability{ + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, + }, + }, + } + + condFn := func() (done bool, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + resp, err := nodeClient.NodePublishVolume(ctx, req) + if err != nil && strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + klog.Errorf("context deadline exceeded; retrying: %v", err) + return false, nil + } + if resp != nil { + return true, nil + } + return false, fmt.Errorf("response from `NodePublishVolume` is nil") + } + + err = wait.PollImmediate( + timeout, + 30*time.Second, + condFn) + assert.NoError(t, err) + + resp, err := http.Get("http://:8080/metrics") + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + b1, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + respBody := string(b1) + assert.Contains(t, respBody, metrics.ImagePullTimeKey) + assert.Contains(t, respBody, metrics.ImageMountTimeKey) + assert.Contains(t, respBody, metrics.OperationErrorsCountKey) + + // give some time before stopping the server + time.Sleep(5 * time.Second) + + // unmount if the volume is already mounted + c, ca := context.WithTimeout(context.Background(), time.Second*10) + defer ca() + + err = mounter.Unmount(c, volId, backend.MountTarget(targetPath)) + assert.NoError(t, err) } +type testSecretStore struct{} + func (t *testSecretStore) GetDockerKeyring(ctx context.Context, secrets map[string]string) (credentialprovider.DockerKeyring, error) { return credentialprovider.UnionDockerKeyring{credentialprovider.NewDockerKeyring()}, nil } diff --git a/docs/automation.md b/docs/automation.md new file mode 100644 index 00000000..12babf34 --- /dev/null +++ b/docs/automation.md @@ -0,0 +1,29 @@ +# Automation + +There are a few [GitHub Workflows](https://docs.github.com/en/actions/using-workflows/about-workflows#about-workflows) that run on this repository. + +## Build +- [![Create and publish the container image.](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/images-build-and-push.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/images-build-and-push.yaml) + - On the main repository - `warm-metal/container-image-csi-driver`, builds and pushes the container image to Docker Hub [`warmmetal/csi-image`](https://hub.docker.com/r/warmmetal/csi-image) + - On any forks, builds and pushes the container image to `ghcr.io/` + +## Tests + +- [![backward-compatibility-5mins](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/backward-compatibility.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/backward-compatibility.yaml) +- [![containerd-11mins](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/containerd.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/containerd.yaml) +- [![cri-o-10mins](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/cri-o.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/cri-o.yaml) +- [![restart-ds-containerd-5mins](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/restart-ds-containerd.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/restart-ds-containerd.yaml) +- [![restart-ds-crio-8mins](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/restart-ds-crio.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/restart-ds-crio.yaml) +- [![test-metrics-5m](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/metrics.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/metrics.yaml) + +## Maintenance + +- [![Delete old container images](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/images-cleanup.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/images-cleanup.yaml) + - Deletes all `ghcr.io/` image tags, expect `latest` or any semver tags. This workflow will run on forks only. +- [![Close stale issues and PRs](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/stale.yaml/badge.svg)](https://github.com/warm-metal/container-image-csi-driver/actions/workflows/stale.yaml) + - [Create a repository variable](https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-a-repository) `DEBUG_ONLY` with value `true` to run the action in dry-run mode. + - Marks issues or PRs as stale after 30 days and closes them after 7 days, except those labeled with any of of the following + - `awaiting-approval` + - `work-in-progress` + - `help-wanted` +- [Dependabot](../.github/dependabot.yml) - [Dependabot](https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide#about-dependabot) is currently being used to [keep the GitHub Actions up to date](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot) diff --git a/docs/maintainer-workflow.md b/docs/maintainer-workflow.md new file mode 100644 index 00000000..c666fc76 --- /dev/null +++ b/docs/maintainer-workflow.md @@ -0,0 +1,71 @@ +# Maintainer guide + +Welcome to the maintainer guide for our project! +As a project maintainer, you play a crucial role in ensuring the success and health of the project. +Below are key responsibilities and guidelines to help you manage the project effectively. + +## Key responsibilities + +### Manage Issues +1. Issue Triage + - Establish a regular schedule for reviewing and triaging open issues. + - Close irrelevant issues, and categorize and prioritize others using milestones. + +1. Acknowledging an Issue + - Review new issues promptly to address major blockers. + - Leave a comment on the issue to show appreciation for the creator's time and efforts. + - Request additional details if the issue description is unclear. + +1. Labelling Issues + - Add appropriate labels to convey the status of each issue. + - Stalebot ignores issues labeled `awaiting-inputs`, `work-in-progress`, and `help-wanted`. See [automation docs](docs/automation.md) + - Guidelines for specific labels: + - `help-wanted`: Use for issues that we want to work on but haven't been picked up. + - `work-in-progress`: Indicate ongoing work without a corresponding pull request. + - `wontfix`/`invalid`: Always add with a clear explanation. + +1. Assigning an Issue + - Assign issues to contributors who show interest or agree to work on them. + - If someone has started working on an issue, assign it to them for visibility. + +### Manage Pull Requests (PRs) +1. PR review + - Hold off on reviewing until the contributor indicates readiness, unless explicitly asked otherwise. + - Label PRs with `awaiting-inputs` if help is needed to understand the changes. + - Ensure that new features or changes are properly documented. + - Ensure that adequate tests are added (or already exists), create a separate issue if required. + +1. PR approval and merge + - Contributors cannot merge PRs, assist them in the merging process. + - Ensure Continuous Integration (CI) passes before merging, except for non-code/test changes. + - [Approve CI builds for first time contributors](https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks#approving-workflow-runs-on-a-pull-request-from-a-public-fork). + +1. Labeling PRs + - Stalebot ignores PRs labeled `awaiting-inputs`, `work-in-progress`, and `help-wanted`. + - It's a good practice to add appropriate labels on PRs. + +### Manage Releases +- Choose the correct tag based on major, minor, or patch fixes using [semver versioning](https://semver.org/). +- Aim to create releases promptly to minimize user wait times. +- If possible, update releases with informative release notes highlighting breaking changes, major features, and bug fixes. + +### Arrange and Drive Community Meetings +- Attend community meetings to engage with contributors and users. +- Promptly address action items from meeting notes. +- Foster a welcoming, open-minded, respectful, and friendly environment. + +### Ensure Code of Conduct Compliance +- Familiarize yourself with the [CNCF code of conduct](https://www.cncf.io/conduct/). +- Enforce the code of conduct among contributors and maintainers, taking necessary actions when required. + +### Keeping the Repository Up to Date +- Periodically review documentation and create issues for any missing pieces. +- Implement automation to reduce inconsistencies and enhance productivity. + +## Reading materials +For further insights, refer to these valuable resources: +- [Best practices for maintainers](https://opensource.guide/best-practices) +- [Practical skills for maintainers](https://www.freecodecamp.org/news/practical-skills-for-open-source-maintainers) +- [Becoming an open source maintainer](https://kentcdodds.com/blog/becoming-an-open-source-project-maintainer) + +**Thank you for your dedication to maintaining our project!** diff --git a/docs/running-locally.md b/docs/running-locally.md new file mode 100644 index 00000000..aecae44c --- /dev/null +++ b/docs/running-locally.md @@ -0,0 +1,49 @@ +# Running Locally + +You have two options: + +1. Use the [Dev Container](#development-container). This is the recommended approach. This can be used with VSCode, the `devcontainer` CLI, or GitHub Codespaces. +1. Install the [requirements](#requirements) on your computer manually. + +## Development Container + +The [development container](https://containers.dev) contains all the tools necessary to work with container-image-csi-driver. + +You can use the development container in a few different ways: + +1. [Visual Studio Code](https://code.visualstudio.com/) with [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). Open the local clone of `container-image-csi-driver` folder in VSCode and it should offer to use the development container automatically. +1. [`devcontainer` CLI](https://github.com/devcontainers/cli). Once installed, the local clone of `container-image-csi-driver` folder and run `devcontainer up --workspace-folder .` followed by `devcontainer exec --workspace-folder . /bin/bash` to get a shell where you can build the code. You can use any editor outside the container to edit code; any changes will be mirrored inside the container. +1. [GitHub Codespaces](https://github.com/codespaces). You can start editing as soon as VSCode is open. + +Once you have entered the container, continue to [Developing Locally](#developing-locally). + +## Requirements + +To build on your own machine without using the Dev Container you will need: + +* A local clone of this repository. +* [Go](https://golang.org/dl/) +* A local Kubernetes cluster ([`k3d`](https://k3d.io/#quick-start), [`kind`](https://kind.sigs.k8s.io/docs/user/quick-start/#installation), or [`minikube`](https://minikube.sigs.k8s.io/docs/start/)) +* [`helm`](https://helm.sh/docs/intro/install/) + +## Developing locally + +_**Note:** Unless specified otherwise, you need to run all commands after changing your working directory to this repository - `cd /path/to/container-image-csi-driver-repository`_ + +1. First, make sure you can connect to the Kubernetes cluster by following the quickstart guide of your chosen local Kubernetes cluster provider. + ``` + $ kubectl get nodes + ``` + Make sure you don't see any errors in your terminal. If do get error(s), please check the quickstart guide or the local Kubernetes cluster provider's documentation on how to get started. + +1. Install the container-image-csi-driver using the helm chart. + ``` + helm upgrade --install wm-csi \ + charts/warm-metal-csi-driver \ + --wait + ``` + +1. You can submit an example for testing using `kubectl`: + ```bash + kubectl create -f sample/ephemeral-volume.yaml + ``` diff --git a/go.mod b/go.mod index 3b5fb756..d93901d2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/warm-metal/csi-driver-image +module github.com/warm-metal/container-image-csi-driver go 1.19 @@ -8,9 +8,11 @@ require ( github.com/containerd/containerd v1.6.8 github.com/containers/storage v1.43.0 github.com/golang/protobuf v1.5.2 + github.com/google/uuid v1.2.0 github.com/mitchellh/go-ps v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.12.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 github.com/warm-metal/csi-drivers v0.5.0-alpha.0.0.20210404173852-9ec9cb097dd2 @@ -67,7 +69,6 @@ require ( github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-intervals v0.0.2 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.2.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -93,7 +94,6 @@ require ( github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.10.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect diff --git a/hack/ci/setup_docker_cluster.sh b/hack/ci/setup_docker_cluster.sh index 916a8ec3..78a07480 100755 --- a/hack/ci/setup_docker_cluster.sh +++ b/hack/ci/setup_docker_cluster.sh @@ -7,7 +7,7 @@ lib::start_cluster_docker $@ set -e echo "Install a private registry" lib::install_private_registry -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i pull docker.io/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i tag --force docker.io/warmmetal/csi-image-test:simple-fs localhost:31000/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i push localhost:31000/warmmetal/csi-image-test:simple-fs --plain-http --user warmmetal:warmmetal +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i pull docker.io/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i tag --force docker.io/warmmetal/container-image-csi-driver-test:simple-fs localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i push localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs --plain-http --user warmmetal:warmmetal set +e diff --git a/hack/ci/setup_private_registry.sh b/hack/ci/setup_private_registry.sh index 138e21c3..c4fa4714 100755 --- a/hack/ci/setup_private_registry.sh +++ b/hack/ci/setup_private_registry.sh @@ -5,8 +5,8 @@ source $(dirname "$0")/../lib/cluster.sh set -e echo "Install a private registry" lib::install_private_registry -docker pull docker.io/warmmetal/csi-image-test:simple-fs -docker tag docker.io/warmmetal/csi-image-test:simple-fs localhost:5000/warmmetal/csi-image-test:simple-fs +docker pull docker.io/warmmetal/container-image-csi-driver-test:simple-fs +docker tag docker.io/warmmetal/container-image-csi-driver-test:simple-fs localhost:5000/warmmetal/container-image-csi-driver-test:simple-fs docker login -u warmmetal -p warmmetal localhost:5000 -docker push localhost:5000/warmmetal/csi-image-test:simple-fs +docker push localhost:5000/warmmetal/container-image-csi-driver-test:simple-fs set +e diff --git a/hack/helper/prepare_containerd_cluster.sh b/hack/helper/prepare_containerd_cluster.sh index 7a4cc2e7..dba39c18 100755 --- a/hack/helper/prepare_containerd_cluster.sh +++ b/hack/helper/prepare_containerd_cluster.sh @@ -11,9 +11,9 @@ lib::install_driver echo "Install a private registry" lib::install_private_registry -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i pull docker.io/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i tag --force docker.io/warmmetal/csi-image-test:simple-fs localhost:31000/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i push localhost:31000/warmmetal/csi-image-test:simple-fs --plain-http --user warmmetal:warmmetal +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i pull docker.io/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i tag --force docker.io/warmmetal/container-image-csi-driver-test:simple-fs localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i push localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs --plain-http --user warmmetal:warmmetal set +x -set +e \ No newline at end of file +set +e diff --git a/hack/integration-test-image/csi-image-test:check-fs.dockerfile b/hack/integration-test-image/container-image-csi-driver-test:check-fs.dockerfile similarity index 100% rename from hack/integration-test-image/csi-image-test:check-fs.dockerfile rename to hack/integration-test-image/container-image-csi-driver-test:check-fs.dockerfile diff --git a/hack/integration-test-image/csi-image-test:simple-fs.dockerfile b/hack/integration-test-image/container-image-csi-driver-test:simple-fs.dockerfile similarity index 100% rename from hack/integration-test-image/csi-image-test:simple-fs.dockerfile rename to hack/integration-test-image/container-image-csi-driver-test:simple-fs.dockerfile diff --git a/hack/integration-test-image/csi-image-test:stat-fs.dockerfile b/hack/integration-test-image/container-image-csi-driver-test:stat-fs.dockerfile similarity index 100% rename from hack/integration-test-image/csi-image-test:stat-fs.dockerfile rename to hack/integration-test-image/container-image-csi-driver-test:stat-fs.dockerfile diff --git a/hack/integration-test-image/csi-image-test:write-check.dockerfile b/hack/integration-test-image/container-image-csi-driver-test:write-check.dockerfile similarity index 60% rename from hack/integration-test-image/csi-image-test:write-check.dockerfile rename to hack/integration-test-image/container-image-csi-driver-test:write-check.dockerfile index d09e92f2..084f776f 100644 --- a/hack/integration-test-image/csi-image-test:write-check.dockerfile +++ b/hack/integration-test-image/container-image-csi-driver-test:write-check.dockerfile @@ -1,10 +1,10 @@ FROM docker.io/library/golang:1.16-alpine as builder -WORKDIR /go/src/csi-image-test +WORKDIR /go/src/container-image-csi-driver-test COPY write_check.go . RUN GO111MODULE=off go build -o write_check FROM alpine:3 ENV TARGET="" WORKDIR / -COPY --from=builder /go/src/csi-image-test/write_check . +COPY --from=builder /go/src/container-image-csi-driver-test/write_check . ENTRYPOINT ["/write_check"] diff --git a/hack/integration-test-image/kind-node-image/Dockerfile b/hack/integration-test-image/kind-node-image/Dockerfile index 74b98e81..a63b68fb 100644 --- a/hack/integration-test-image/kind-node-image/Dockerfile +++ b/hack/integration-test-image/kind-node-image/Dockerfile @@ -1,2 +1,2 @@ -FROM docker.io/warmmetal/kindest-node-crio:v1.25.2 +FROM docker.io/warmmetal/kindest-node-crio:v1.25.2-updated COPY crio.conf /etc/crio/crio.conf diff --git a/hack/lib/cluster.sh b/hack/lib/cluster.sh index f8ecbea7..4ae2504d 100755 --- a/hack/lib/cluster.sh +++ b/hack/lib/cluster.sh @@ -4,7 +4,7 @@ source $(dirname "${BASH_SOURCE[0]}")/utils.sh function lib::start_cluster_containerd() { local version=${1:-stable} - minikube start -p csi-image-test \ + minikube start -p container-image-csi-driver-test \ --kubernetes-version=${version} \ --container-runtime=containerd \ --insecure-registry=localhost:31000 @@ -12,7 +12,7 @@ function lib::start_cluster_containerd() { function lib::start_cluster_crio() { local version=${1:-stable} - minikube start -p csi-image-test \ + minikube start -p container-image-csi-driver-test \ --kubernetes-version=${version} \ --container-runtime=cri-o \ --insecure-registry=localhost:31000 @@ -20,7 +20,7 @@ function lib::start_cluster_crio() { function lib::start_cluster_docker() { local version=${1:-stable} - minikube start -p csi-image-test \ + minikube start -p container-image-csi-driver-test \ --kubernetes-version=${version} \ --container-runtime=docker \ --insecure-registry=localhost:31000 @@ -101,7 +101,7 @@ function lib::install_private_registry() { if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' 'private-registry')" = 'null' ]; then docker network connect "kind" "private-registry" fi - + cat < | while read job` creates a subshell + # because of which increment in `total_failed` is never reflected outside the loop + # more info: https://unix.stackexchange.com/questions/402750/modify-global-variable-in-while-loop + + [[ total_failed -eq 0 ]] && kubectl delete -f "$manifest" } function lib::run_failed_test_job() { diff --git a/pkg/backend/containerd/containerd.go b/pkg/backend/containerd/containerd.go index 6227aaf0..8dfcdd3d 100644 --- a/pkg/backend/containerd/containerd.go +++ b/pkg/backend/containerd/containerd.go @@ -13,7 +13,7 @@ import ( "github.com/containerd/containerd/reference/docker" "github.com/containerd/containerd/snapshots" "github.com/opencontainers/image-spec/identity" - "github.com/warm-metal/csi-driver-image/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" "k8s.io/klog/v2" ) @@ -22,7 +22,7 @@ type snapshotMounter struct { cli *containerd.Client } -func NewMounter(socketPath string) *backend.SnapshotMounter { +func NewMounter(socketPath string) backend.Mounter { c, err := containerd.New(socketPath, containerd.WithDefaultNamespace("k8s.io")) if err != nil { klog.Fatalf("containerd connection is broken because the mounted unix socket somehow dose not work,"+ diff --git a/pkg/backend/crio/crio.go b/pkg/backend/crio/crio.go index 6c7e3a4d..f7bb9b23 100644 --- a/pkg/backend/crio/crio.go +++ b/pkg/backend/crio/crio.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "net" "net/http" "reflect" @@ -14,7 +13,7 @@ import ( "github.com/containerd/containerd/reference/docker" "github.com/containers/storage" "github.com/containers/storage/types" - "github.com/warm-metal/csi-driver-image/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" "k8s.io/klog/v2" k8smount "k8s.io/utils/mount" ) diff --git a/pkg/backend/mounter.go b/pkg/backend/mounter.go index 88717589..59a126fe 100644 --- a/pkg/backend/mounter.go +++ b/pkg/backend/mounter.go @@ -12,7 +12,7 @@ import ( ) type SnapshotMounter struct { - runtime ContainerRuntime + runtime ContainerRuntimeMounter guard sync.Mutex // mapping from targets to key of read-only snapshots @@ -21,7 +21,7 @@ type SnapshotMounter struct { roSnapshotTargetsMap map[SnapshotKey]map[MountTarget]struct{} } -func NewMounter(runtime ContainerRuntime) *SnapshotMounter { +func NewMounter(runtime ContainerRuntimeMounter) *SnapshotMounter { mounter := &SnapshotMounter{ runtime: runtime, targetRoSnapshotMap: make(map[MountTarget]SnapshotKey), @@ -173,7 +173,7 @@ func (s *SnapshotMounter) Mount( klog.Fatalf("invalid image id of image %q", image) } - key = genSnapshotKey(imageID) + key = GenSnapshotKey(imageID) klog.Infof("refer read-only snapshot of image %q with key %q", image, key) if err := s.refROSnapshot(ctx, target, imageID, key, createSnapshotMetaData(target)); err != nil { return err @@ -189,7 +189,7 @@ func (s *SnapshotMounter) Mount( }() } else { // For read-write volumes, they must be ephemeral volumes, that which volumeIDs are unique strings. - key = genSnapshotKey(volumeId) + key = GenSnapshotKey(volumeId) klog.Infof("create read-write snapshot of image %q with key %q", image, key) if err := s.runtime.PrepareRWSnapshot(ctx, imageID, key, nil); err != nil { return err @@ -221,13 +221,13 @@ func (s *SnapshotMounter) Unmount(ctx context.Context, volumeId string, target M klog.Infof("delete the read-write snapshot") // Must be a read-write snapshot - return s.runtime.DestroySnapshot(ctx, genSnapshotKey(volumeId)) + return s.runtime.DestroySnapshot(ctx, GenSnapshotKey(volumeId)) } func (s *SnapshotMounter) ImageExists(ctx context.Context, image docker.Named) bool { return s.runtime.ImageExists(ctx, image) } -func genSnapshotKey(parent string) SnapshotKey { +func GenSnapshotKey(parent string) SnapshotKey { return SnapshotKey(fmt.Sprintf("csi-image.warm-metal.tech-%s", parent)) } diff --git a/pkg/backend/runtime.go b/pkg/backend/runtime.go index b50d0813..28b1a548 100644 --- a/pkg/backend/runtime.go +++ b/pkg/backend/runtime.go @@ -13,7 +13,8 @@ type MountOptions struct { type SnapshotKey string type MountTarget string -type ContainerRuntime interface { +// ContainerRuntimeMounter is a container runtime specific interface +type ContainerRuntimeMounter interface { Mount(ctx context.Context, key SnapshotKey, target MountTarget, ro bool) error Unmount(ctx context.Context, target MountTarget) error @@ -43,3 +44,16 @@ type ContainerRuntime interface { // The snapshot key must also be saved in the returned map with the key "FakeMetaDataSnapshotKey". ListSnapshots(ctx context.Context) ([]SnapshotMetadata, error) } + +// Mounter is a generic interface used for mounting images +type Mounter interface { + // Mount mounts a specific image + Mount( + ctx context.Context, volumeId string, target MountTarget, image docker.Named, ro bool) (err error) + + // Unmount unmounts a specific image + Unmount(ctx context.Context, volumeId string, target MountTarget) error + + // ImageExists checks if the image already exists on the local machine + ImageExists(ctx context.Context, image docker.Named) bool +} diff --git a/pkg/cri/client.go b/pkg/cri/client.go index c10b19d2..12ee6f6e 100644 --- a/pkg/cri/client.go +++ b/pkg/cri/client.go @@ -5,7 +5,7 @@ import ( "time" "google.golang.org/grpc" - cri "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + cri "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/kubelet/util" ) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 00000000..c4104ad8 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/klog/v2" +) + +const Async = "async" +const Sync = "sync" +const ImagePullTimeKey = "pull_duration_seconds" +const ImageMountTimeKey = "mount_duration_seconds" +const OperationErrorsCountKey = "operation_errors_total" + +var ImagePullTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "warm_metal", + Name: ImagePullTimeKey, + Help: "The time it took to pull an image", + Buckets: []float64{0, 1, 5, 10, 15, 30, 60, 120, 180}, + }, + []string{"operation_type"}, +) + +var ImageMountTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "warm_metal", + Name: ImageMountTimeKey, + Help: "The time it took to mount an image", + Buckets: []float64{0, 1, 5, 10, 15, 30, 60, 120, 180}, + }, + []string{"operation_type"}, +) + +var OperationErrorsCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "warm_metal", + Name: OperationErrorsCountKey, + Help: "Cumulative number of operation (pull,mount,unmount) errors in the driver", + }, + []string{"operation_type"}, +) + +func RegisterMetrics() *prometheus.Registry { + reg := prometheus.NewRegistry() + reg.MustRegister(ImagePullTime) + reg.MustRegister(ImageMountTime) + reg.MustRegister(OperationErrorsCount) + + return reg +} + +func StartMetricsServer(reg *prometheus.Registry) { + go func() { + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg})) + klog.Info("serving internal metrics at port 8080") + klog.Fatal(http.ListenAndServe(":8080", nil)) + }() +} diff --git a/pkg/mountexecutor/mountexecutor.go b/pkg/mountexecutor/mountexecutor.go index 5d2e42fd..118fb563 100644 --- a/pkg/mountexecutor/mountexecutor.go +++ b/pkg/mountexecutor/mountexecutor.go @@ -8,9 +8,10 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/containerd/containerd/reference/docker" - "github.com/warm-metal/csi-driver-image/pkg/backend" - "github.com/warm-metal/csi-driver-image/pkg/mountstatus" - "github.com/warm-metal/csi-driver-image/pkg/pullstatus" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/metrics" + "github.com/warm-metal/container-image-csi-driver/pkg/mountstatus" + "github.com/warm-metal/container-image-csi-driver/pkg/pullstatus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) @@ -24,7 +25,7 @@ const ( // MountExecutorOptions are options passed to mount executor type MountExecutorOptions struct { AsyncMount bool - Mounter *backend.SnapshotMounter + Mounter backend.Mounter } // MountOptions are options for a single mount request @@ -36,13 +37,14 @@ type MountOptions struct { TargetPath string VolumeCapability *csi.VolumeCapability ReadOnly bool + Logger klog.Logger } // MountExecutor executes mount type MountExecutor struct { asyncMount bool mutex *sync.Mutex - mounter *backend.SnapshotMounter + mounter backend.Mounter asyncErrs map[docker.Named]error } @@ -57,11 +59,12 @@ func NewMountExecutor(o *MountExecutorOptions) *MountExecutor { // StartMounting starts the mounting func (m *MountExecutor) StartMounting(o *MountOptions) error { - - if pullstatus.Get(o.NamedRef) != pullstatus.Pulled || mountstatus.Get(o.VolumeId) == mountstatus.StillMounting { - klog.Infof("image '%s' hasn't been pulled yet (status: %s) or volume is still mounting (status: %s)", - o.NamedRef.Name(), - pullstatus.Get(o.NamedRef), mountstatus.Get(o.VolumeId)) + o.Logger.Info("Mounting image", "image", o.NamedRef.Name()) + if pullstatus.Get(o.NamedRef) != pullstatus.Pulled || mountstatus.Get(o.TargetPath) == mountstatus.StillMounting { + o.Logger.Info("Could not mount image because image hasn't finshed pulling or volume is still mounting", + "image", o.NamedRef.Name(), + "pull-status", pullstatus.Get(o.NamedRef), + "mount-status", mountstatus.Get(o.TargetPath)) return nil } @@ -70,12 +73,18 @@ func (m *MountExecutor) StartMounting(o *MountOptions) error { o.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY if !m.asyncMount { - mountstatus.Update(o.VolumeId, mountstatus.StillMounting) + mountstatus.Update(o.TargetPath, mountstatus.StillMounting) + startTime := time.Now() if err := m.mounter.Mount(o.Context, o.VolumeId, backend.MountTarget(o.TargetPath), o.NamedRef, ro); err != nil { - mountstatus.Update(o.VolumeId, mountstatus.Errored) + o.Logger.Error(err, "mount error") + metrics.OperationErrorsCount.WithLabelValues("StartMounting").Inc() + mountstatus.Update(o.TargetPath, mountstatus.Errored) return err } - mountstatus.Update(o.VolumeId, mountstatus.Mounted) + elapsed := time.Since(startTime) + metrics.ImageMountTime.WithLabelValues(metrics.Sync).Observe(elapsed.Seconds()) + o.Logger.Info("Finished mounting", "image", o.NamedRef.Name(), "mount-duration", elapsed) + mountstatus.Update(o.TargetPath, mountstatus.Mounted) return nil } @@ -85,14 +94,19 @@ func (m *MountExecutor) StartMounting(o *MountOptions) error { ctx, cancel := context.WithTimeout(context.Background(), mountCtxTimeout) defer cancel() - mountstatus.Update(o.VolumeId, mountstatus.StillMounting) + mountstatus.Update(o.TargetPath, mountstatus.StillMounting) + startTime := time.Now() if err := m.mounter.Mount(ctx, o.VolumeId, backend.MountTarget(o.TargetPath), o.NamedRef, ro); err != nil { - klog.Errorf("mount err: %v", err.Error()) - mountstatus.Update(o.VolumeId, mountstatus.Errored) + o.Logger.Error(err, "mount error") + metrics.OperationErrorsCount.WithLabelValues("StartMounting").Inc() + mountstatus.Update(o.TargetPath, mountstatus.Errored) m.asyncErrs[o.NamedRef] = fmt.Errorf("err: %v: %v", err, m.asyncErrs[o.NamedRef]) return } - mountstatus.Update(o.VolumeId, mountstatus.Mounted) + elapsed := time.Since(startTime) + metrics.ImageMountTime.WithLabelValues(metrics.Async).Observe(elapsed.Seconds()) + o.Logger.Info("Finished mounting", "image", o.NamedRef.Name(), "mount-duration", elapsed) + mountstatus.Update(o.TargetPath, mountstatus.Mounted) }() return nil @@ -109,7 +123,7 @@ func (m *MountExecutor) WaitForMount(o *MountOptions) error { } mountCondFn := func() (done bool, err error) { - if mountstatus.Get(o.VolumeId) == mountstatus.Mounted { + if mountstatus.Get(o.TargetPath) == mountstatus.Mounted { return true, nil } if m.asyncErrs[o.NamedRef] != nil { diff --git a/pkg/pullexecutor/pullexecutor.go b/pkg/pullexecutor/pullexecutor.go index 1fe9e589..dbedd65f 100644 --- a/pkg/pullexecutor/pullexecutor.go +++ b/pkg/pullexecutor/pullexecutor.go @@ -8,12 +8,13 @@ import ( "github.com/containerd/containerd/reference/docker" "github.com/pkg/errors" - "github.com/warm-metal/csi-driver-image/pkg/backend" - "github.com/warm-metal/csi-driver-image/pkg/pullstatus" - "github.com/warm-metal/csi-driver-image/pkg/remoteimage" - "github.com/warm-metal/csi-driver-image/pkg/secret" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" + "github.com/warm-metal/container-image-csi-driver/pkg/metrics" + "github.com/warm-metal/container-image-csi-driver/pkg/pullstatus" + "github.com/warm-metal/container-image-csi-driver/pkg/remoteimage" + "github.com/warm-metal/container-image-csi-driver/pkg/secret" "k8s.io/apimachinery/pkg/util/wait" - cri "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + cri "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" ) @@ -28,7 +29,7 @@ type PullExecutorOptions struct { AsyncPull bool ImageServiceClient cri.ImageServiceClient SecretStore secret.Store - Mounter *backend.SnapshotMounter + Mounter backend.Mounter } // PullOptions are the options for a single pull request @@ -39,6 +40,7 @@ type PullOptions struct { PullAlways bool PullSecrets map[string]string Image string + Logger klog.Logger } // PullExecutor executes the pulls @@ -48,7 +50,7 @@ type PullExecutor struct { mutex *sync.Mutex asyncErrs map[docker.Named]error secretStore secret.Store - mounter *backend.SnapshotMounter + mounter backend.Mounter } // NewPullExecutor initializes a new pull executor object @@ -59,12 +61,12 @@ func NewPullExecutor(o *PullExecutorOptions) *PullExecutor { imageSvcClient: o.ImageServiceClient, secretStore: o.SecretStore, mounter: o.Mounter, + asyncErrs: make(map[docker.Named]error), } } // StartPulling starts pulling the image func (m *PullExecutor) StartPulling(o *PullOptions) error { - keyring, err := m.secretStore.GetDockerKeyring(o.Context, o.PullSecrets) if err != nil { return errors.Errorf("unable to fetch keyring: %s", err) @@ -74,12 +76,19 @@ func (m *PullExecutor) StartPulling(o *PullOptions) error { puller := remoteimage.NewPuller(m.imageSvcClient, o.NamedRef, keyring) shouldPull := o.PullAlways || !m.mounter.ImageExists(o.Context, o.NamedRef) if shouldPull { - klog.Infof("pull image %q ", o.Image) + o.Logger.Info("Pulling image", "image", o.Image) pullstatus.Update(o.NamedRef, pullstatus.StillPulling) + startTime := time.Now() if err = puller.Pull(o.Context); err != nil { pullstatus.Update(o.NamedRef, pullstatus.Errored) + metrics.OperationErrorsCount.WithLabelValues("StartPulling").Inc() + o.Logger.Error(err, "Unable to pull image", "image", o.NamedRef) return errors.Errorf("unable to pull image %q: %s", o.NamedRef, err) } + elapsed := time.Since(startTime) + metrics.ImagePullTime.WithLabelValues(metrics.Sync).Observe(elapsed.Seconds()) + size := puller.ImageSize(o.Context) + o.Logger.Info("Finished pulling image", "image", o.Image, "pull-duration", elapsed, "image-size", fmt.Sprintf("%.2f MiB", float64(size)/(1024.0*1024.0))) } pullstatus.Update(o.NamedRef, pullstatus.Pulled) return nil @@ -97,23 +106,32 @@ func (m *PullExecutor) StartPulling(o *PullOptions) error { c, cancel := context.WithTimeout(context.Background(), pullCtxTimeout) defer cancel() - if pullstatus.Get(o.NamedRef) == pullstatus.StillPulling { + if pullstatus.Get(o.NamedRef) == pullstatus.StillPulling || + pullstatus.Get(o.NamedRef) == pullstatus.Pulled { return } puller := remoteimage.NewPuller(m.imageSvcClient, o.NamedRef, keyring) shouldPull := o.PullAlways || !m.mounter.ImageExists(o.Context, o.NamedRef) if shouldPull { - klog.Infof("pull image %q ", o.Image) + o.Logger.Info("Pulling image asynchronously", "image", o.Image) pullstatus.Update(o.NamedRef, pullstatus.StillPulling) + startTime := time.Now() + if err = puller.Pull(c); err != nil { pullstatus.Update(o.NamedRef, pullstatus.Errored) + metrics.OperationErrorsCount.WithLabelValues("StartPulling").Inc() + o.Logger.Error(err, "Unable to pull image", "image", o.Image) m.asyncErrs[o.NamedRef] = fmt.Errorf("unable to pull image %q: %s", o.Image, err) return } - pullstatus.Update(o.NamedRef, pullstatus.Pulled) + elapsed := time.Since(startTime) + metrics.ImagePullTime.WithLabelValues(metrics.Async).Observe(elapsed.Seconds()) + size := puller.ImageSize(o.Context) + o.Logger.Info("Finished pulling image", "image", o.Image, "pull-duration", elapsed, "image-size", fmt.Sprintf("%.2f MiB", float64(size)/(1024.0*1024.0))) } + pullstatus.Update(o.NamedRef, pullstatus.Pulled) } }() diff --git a/pkg/remoteimage/pull.go b/pkg/remoteimage/pull.go index 9df21da2..7ca29d5a 100644 --- a/pkg/remoteimage/pull.go +++ b/pkg/remoteimage/pull.go @@ -5,12 +5,13 @@ import ( "github.com/containerd/containerd/reference/docker" utilerrors "k8s.io/apimachinery/pkg/util/errors" - cri "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + cri "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/kubernetes/pkg/credentialprovider" ) type Puller interface { Pull(context.Context) error + ImageSize(context.Context) int } func NewPuller(imageSvc cri.ImageServiceClient, image docker.Named, @@ -28,6 +29,16 @@ type puller struct { keyring credentialprovider.DockerKeyring } +// Returns the compressed size of the image that was pulled in bytes +// see https://github.com/containerd/containerd/issues/9261 +func (p puller) ImageSize(ctx context.Context) int { + imageSpec := &cri.ImageSpec{Image: p.image.String()} + imageStatusResponse, _ := p.imageSvc.ImageStatus(ctx, &cri.ImageStatusRequest{ + Image: imageSpec, + }) + return int(imageStatusResponse.Image.Size_) +} + func (p puller) Pull(ctx context.Context) (err error) { repo := p.image.Name() imageSpec := &cri.ImageSpec{Image: p.image.String()} diff --git a/pkg/remoteimage/pull_test.go b/pkg/remoteimage/pull_test.go index 52a1d18f..d9b38321 100644 --- a/pkg/remoteimage/pull_test.go +++ b/pkg/remoteimage/pull_test.go @@ -6,8 +6,8 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/warm-metal/csi-driver-image/pkg/cri" - "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "github.com/warm-metal/container-image-csi-driver/pkg/cri" + v1 "k8s.io/cri-api/pkg/apis/runtime/v1" ) // Check test/integration/node-server/README.md for how to run this test correctly @@ -20,8 +20,8 @@ func TestPull(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, criClient) - r, err := criClient.PullImage(context.Background(), &v1alpha2.PullImageRequest{ - Image: &v1alpha2.ImageSpec{ + r, err := criClient.PullImage(context.Background(), &v1.PullImageRequest{ + Image: &v1.ImageSpec{ Image: testImage, }, }) diff --git a/pkg/test/utils/utils.go b/pkg/test/utils/utils.go new file mode 100644 index 00000000..72ad1fbd --- /dev/null +++ b/pkg/test/utils/utils.go @@ -0,0 +1,111 @@ +package utils + +import ( + "context" + "fmt" + "time" + + "github.com/containerd/containerd/reference/docker" + "github.com/warm-metal/container-image-csi-driver/pkg/backend" + "google.golang.org/grpc" + criapi "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +type MockImageServiceClient struct { + PulledImages map[string]bool + ImagePullTime time.Duration +} + +type MockMounter struct { + ImageSvcClient MockImageServiceClient + Mounted map[string]bool +} + +const hundredMB = 104857600 + +func (m *MockMounter) Mount( + ctx context.Context, volumeId string, target backend.MountTarget, image docker.Named, ro bool) (err error) { + m.Mounted[volumeId] = true + return nil +} + +// Unmount unmounts a specific image +func (m *MockMounter) Unmount(ctx context.Context, volumeId string, target backend.MountTarget) error { + if m.Mounted[volumeId] { + delete(m.Mounted, volumeId) + return nil + } + return fmt.Errorf("image mount not found") +} + +// ImageExists checks if the image already exists on the local machine +func (m *MockMounter) ImageExists(ctx context.Context, image docker.Named) bool { + return m.ImageSvcClient.PulledImages[image.Name()] +} + +func (c *MockImageServiceClient) ListImages(ctx context.Context, in *criapi.ListImagesRequest, opts ...grpc.CallOption) (*criapi.ListImagesResponse, error) { + resp := new(criapi.ListImagesResponse) + resp.Images = []*criapi.Image{} + + for k := range c.PulledImages { + resp.Images = append(resp.Images, &criapi.Image{ + Id: k, + // 100MB + Size_: hundredMB, + Spec: &criapi.ImageSpec{ + Image: k, + }, + }) + } + + return resp, nil +} + +func (c *MockImageServiceClient) ImageStatus(ctx context.Context, in *criapi.ImageStatusRequest, opts ...grpc.CallOption) (*criapi.ImageStatusResponse, error) { + resp := new(criapi.ImageStatusResponse) + resp.Image = &criapi.Image{ + Id: in.Image.Image, + Size_: hundredMB, + Spec: &criapi.ImageSpec{ + Image: in.Image.Image, + }, + } + return resp, nil +} + +func (c *MockImageServiceClient) PullImage(ctx context.Context, in *criapi.PullImageRequest, opts ...grpc.CallOption) (*criapi.PullImageResponse, error) { + resp := new(criapi.PullImageResponse) + resp.ImageRef = in.Image.Image + time.Sleep(c.ImagePullTime) + + return resp, nil +} + +func (c *MockImageServiceClient) RemoveImage(ctx context.Context, in *criapi.RemoveImageRequest, opts ...grpc.CallOption) (*criapi.RemoveImageResponse, error) { + resp := new(criapi.RemoveImageResponse) + delete(c.PulledImages, in.Image.Image) + return resp, nil +} + +func (c *MockImageServiceClient) ImageFsInfo(ctx context.Context, in *criapi.ImageFsInfoRequest, opts ...grpc.CallOption) (*criapi.ImageFsInfoResponse, error) { + resp := new(criapi.ImageFsInfoResponse) + resp.ImageFilesystems = []*criapi.FilesystemUsage{} + + for _ = range c.PulledImages { + resp.ImageFilesystems = append(resp.ImageFilesystems, &criapi.FilesystemUsage{ + Timestamp: time.Now().Unix(), + FsId: &criapi.FilesystemIdentifier{ + Mountpoint: "target", + }, + UsedBytes: &criapi.UInt64Value{ + Value: hundredMB, + }, + InodesUsed: &criapi.UInt64Value{ + // random value + Value: 10, + }, + }) + } + + return resp, nil +} diff --git a/sample/ephemeral-volume.yaml b/sample/ephemeral-volume.yaml index 3019b608..2039a3b5 100644 --- a/sample/ephemeral-volume.yaml +++ b/sample/ephemeral-volume.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: ephemeral-volume - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -22,7 +22,7 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" # # set pullAlways if you want to ignore local images # pullAlways: "true" # # set secret if the image is private diff --git a/sample/install/cri-o.yaml b/sample/install/cri-o.yaml index 15a75c45..a6fe4d49 100644 --- a/sample/install/cri-o.yaml +++ b/sample/install/cri-o.yaml @@ -126,7 +126,7 @@ spec: - mountPath: /run/containers/storage mountPropagation: Bidirectional name: crio-run-root - hostNetwork: true + hostNetwork: false serviceAccountName: csi-image-warm-metal volumes: - hostPath: diff --git a/sample/install/k3s-containerd.yaml b/sample/install/k3s-containerd.yaml index 9580348b..65416d92 100644 --- a/sample/install/k3s-containerd.yaml +++ b/sample/install/k3s-containerd.yaml @@ -117,7 +117,7 @@ spec: - mountPath: /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs mountPropagation: Bidirectional name: snapshot-root-0 - hostNetwork: true + hostNetwork: false serviceAccountName: csi-image-warm-metal volumes: - hostPath: diff --git a/sample/install/minikube-containerd.yaml b/sample/install/minikube-containerd.yaml index e3b469e7..ea2c4aff 100644 --- a/sample/install/minikube-containerd.yaml +++ b/sample/install/minikube-containerd.yaml @@ -123,7 +123,7 @@ spec: - mountPath: /mnt/vda1/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs mountPropagation: Bidirectional name: snapshot-root-0 - hostNetwork: true + hostNetwork: false serviceAccountName: csi-image-warm-metal volumes: - hostPath: diff --git a/sample/install/snap-microk8s-containerd.yaml b/sample/install/snap-microk8s-containerd.yaml index 230fa679..70472fc5 100644 --- a/sample/install/snap-microk8s-containerd.yaml +++ b/sample/install/snap-microk8s-containerd.yaml @@ -117,7 +117,7 @@ spec: - mountPath: /var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs mountPropagation: Bidirectional name: snapshot-root-0 - hostNetwork: true + hostNetwork: false serviceAccountName: csi-image-warm-metal volumes: - hostPath: diff --git a/sample/pre-provisioned-pv.yaml b/sample/pre-provisioned-pv.yaml index c4daa926..1da24731 100644 --- a/sample/pre-provisioned-pv.yaml +++ b/sample/pre-provisioned-pv.yaml @@ -7,7 +7,7 @@ provisioner: csi-image.warm-metal.tech apiVersion: v1 kind: PersistentVolume metadata: - name: pv-test-csi-image-test-simple-fs + name: pv-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech capacity: @@ -17,7 +17,7 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" # volumeAttributes: # # set pullAlways if you want to ignore local images # pullAlways: "true" @@ -28,7 +28,7 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: pvc-test-csi-image-test-simple-fs + name: pvc-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech accessModes: @@ -36,7 +36,7 @@ spec: resources: requests: storage: 5Gi - volumeName: pv-test-csi-image-test-simple-fs + volumeName: pv-test-container-image-csi-driver-test-simple-fs --- apiVersion: batch/v1 kind: Job @@ -49,7 +49,7 @@ spec: spec: containers: - name: pre-provisioned-pv - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -60,5 +60,5 @@ spec: volumes: - name: target persistentVolumeClaim: - claimName: pvc-test-csi-image-test-simple-fs + claimName: pvc-test-container-image-csi-driver-test-simple-fs backoffLimit: 0 diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile index e9cd0b07..a9a026da 100644 --- a/test/e2e/Dockerfile +++ b/test/e2e/Dockerfile @@ -1,6 +1,6 @@ FROM docker.io/library/golang:1.16 -WORKDIR /go/src/csi-driver-image +WORKDIR /go/src/container-image-csi-driver COPY go.mod go.sum ./ RUN go mod download @@ -11,4 +11,4 @@ COPY id_rsa* /root/.ssh/ COPY *.go . -ENTRYPOINT go run . \ No newline at end of file +ENTRYPOINT go run . diff --git a/test/e2e/driver.go b/test/e2e/driver.go index 16e5a793..5229c60c 100644 --- a/test/e2e/driver.go +++ b/test/e2e/driver.go @@ -9,11 +9,10 @@ import ( "k8s.io/kubernetes/test/e2e/storage/testsuites" ) -type driver struct { -} +type driver struct{} func (d driver) GetVolume(config *testsuites.PerTestConfig, volumeNumber int) (attributes map[string]string, shared bool, readOnly bool) { - return map[string]string{"image": "docker.io/warmmetal/csi-image-test:simple-fs"}, true, false + return map[string]string{"image": "docker.io/warmmetal/container-image-csi-driver-test:simple-fs"}, true, false } func (d driver) GetCSIDriverName(config *testsuites.PerTestConfig) string { @@ -28,7 +27,7 @@ func (d driver) GetPersistentVolumeSource(readOnly bool, fsType string, testVolu return &v1.PersistentVolumeSource{ CSI: &v1.CSIPersistentVolumeSource{ Driver: "csi-image.warm-metal.tech", - VolumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs", + VolumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs", ReadOnly: true, }, }, &v1.VolumeNodeAffinity{ @@ -55,14 +54,13 @@ func (d driver) GetVolumeSource(readOnly bool, fsType string, testVolume testsui Driver: "csi-image.warm-metal.tech", ReadOnly: &ro, VolumeAttributes: map[string]string{ - "image": "docker.io/warmmetal/csi-image-test:simple-fs", + "image": "docker.io/warmmetal/container-image-csi-driver-test:simple-fs", }, }, } } -type imageVol struct { -} +type imageVol struct{} func (i imageVol) DeleteVolume() { } diff --git a/test/e2e/manifest.yaml b/test/e2e/manifest.yaml index fcf65627..9766cb07 100644 --- a/test/e2e/manifest.yaml +++ b/test/e2e/manifest.yaml @@ -1,36 +1,36 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: csi-driver-image-e2e-test + name: container-image-csi-driver-e2e-test namespace: cliapp-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: csi-driver-image-e2e-test + name: container-image-csi-driver-e2e-test roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount - name: csi-driver-image-e2e-test + name: container-image-csi-driver-e2e-test namespace: cliapp-system --- apiVersion: batch/v1 kind: Job metadata: - name: csi-driver-image-e2e-test + name: container-image-csi-driver-e2e-test namespace: cliapp-system spec: template: metadata: - name: csi-driver-image-e2e-test + name: container-image-csi-driver-e2e-test spec: - serviceAccountName: csi-driver-image-e2e-test + serviceAccountName: container-image-csi-driver-e2e-test containers: - - name: csi-driver-image-e2e-test - image: local.test/csi-driver-image-test:e2e + - name: container-image-csi-driver-e2e-test + image: local.test/container-image-csi-driver-test:e2e env: - name: KUBE_SSH_USER value: docker diff --git a/test/integration/backward-compatability.sh b/test/integration/backward-compatability.sh index c75fb4de..2a48c760 100755 --- a/test/integration/backward-compatability.sh +++ b/test/integration/backward-compatability.sh @@ -2,7 +2,7 @@ set -e source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/cluster.sh -lib::install_driver_from_manifest_file 'https://raw.githubusercontent.com/warm-metal/csi-driver-image/v0.4.2/install/cri-containerd.yaml' +lib::install_driver_from_manifest_file 'https://raw.githubusercontent.com/warm-metal/container-image-csi-driver/v0.4.2/install/cri-containerd.yaml' TestBase=$(dirname "${BASH_SOURCE[0]}") kubectl apply -f "${TestBase}/compatible-manifests/ephemeral-volume.yaml" @@ -11,7 +11,7 @@ kubectlwait default compatible-ephemeral-volume kubectl apply -f "${TestBase}/compatible-manifests/pre-provisioned-pv.yaml" kubectlwait default compatible-pre-provisioned-pv -kubectl delete --ignore-not-found -f 'https://raw.githubusercontent.com/warm-metal/csi-driver-image/v0.4.2/install/cri-containerd.yaml' +kubectl delete --ignore-not-found -f 'https://raw.githubusercontent.com/warm-metal/container-image-csi-driver/v0.4.2/install/cri-containerd.yaml' export VALUE_FILE=$(dirname "${BASH_SOURCE[0]}")/../../charts/warm-metal-csi-driver/values.yaml export IMAGE_TAG=$(git rev-parse --short HEAD) @@ -35,4 +35,4 @@ kubectl delete -f "${TestBase}/compatible-manifests/ephemeral-volume.yaml" kubectl delete --ignore-not-found -f "${TestBase}/compatible-manifests/pre-provisioned-pv.yaml" helm uninstall -n kube-system ${HELM_NAME} --wait -set +e \ No newline at end of file +set +e diff --git a/test/integration/compatible-manifests/ephemeral-volume.yaml b/test/integration/compatible-manifests/ephemeral-volume.yaml index 9a743a8f..7e9416b0 100644 --- a/test/integration/compatible-manifests/ephemeral-volume.yaml +++ b/test/integration/compatible-manifests/ephemeral-volume.yaml @@ -19,12 +19,12 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" - name: ro csi: driver: csi-image.warm-metal.tech readOnly: true volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" - pullAlways: "true" \ No newline at end of file + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" + pullAlways: "true" diff --git a/test/integration/compatible-manifests/pre-provisioned-pv.yaml b/test/integration/compatible-manifests/pre-provisioned-pv.yaml index e837e15a..ce797c77 100644 --- a/test/integration/compatible-manifests/pre-provisioned-pv.yaml +++ b/test/integration/compatible-manifests/pre-provisioned-pv.yaml @@ -7,7 +7,7 @@ provisioner: csi-image.warm-metal.tech apiVersion: v1 kind: PersistentVolume metadata: - name: compatible-pv-test-csi-image-test-simple-fs + name: compatible-pv-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech capacity: @@ -17,12 +17,12 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: compatible-pvc-test-csi-image-test-simple-fs + name: compatible-pvc-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech accessModes: @@ -30,7 +30,7 @@ spec: resources: requests: storage: 5Gi - volumeName: compatible-pv-test-csi-image-test-simple-fs + volumeName: compatible-pv-test-container-image-csi-driver-test-simple-fs --- apiVersion: v1 kind: Pod @@ -50,4 +50,4 @@ spec: volumes: - name: target persistentVolumeClaim: - claimName: compatible-pvc-test-csi-image-test-simple-fs + claimName: compatible-pvc-test-container-image-csi-driver-test-simple-fs diff --git a/test/integration/daemon-dependent-manifests/ephemeral-volume-private-with-daemon-secret.yaml b/test/integration/daemon-dependent-manifests/ephemeral-volume-private-with-daemon-secret.yaml index e09e845c..deea939d 100644 --- a/test/integration/daemon-dependent-manifests/ephemeral-volume-private-with-daemon-secret.yaml +++ b/test/integration/daemon-dependent-manifests/ephemeral-volume-private-with-daemon-secret.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: ephemeral-volume-private-with-daemon-secret - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -22,6 +22,6 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "private-registry:5000/warmmetal/csi-image-test:simple-fs" + image: "private-registry:5000/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" backoffLimit: 0 diff --git a/test/integration/daemon-dependent-manifests/pre-provisioned-pv-private-with-daemon-secret.yaml b/test/integration/daemon-dependent-manifests/pre-provisioned-pv-private-with-daemon-secret.yaml index 7a125345..5bc8dcd7 100644 --- a/test/integration/daemon-dependent-manifests/pre-provisioned-pv-private-with-daemon-secret.yaml +++ b/test/integration/daemon-dependent-manifests/pre-provisioned-pv-private-with-daemon-secret.yaml @@ -17,7 +17,7 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "private-registry:5000/warmmetal/csi-image-test:simple-fs" + volumeHandle: "private-registry:5000/warmmetal/container-image-csi-driver-test:simple-fs" --- apiVersion: v1 kind: PersistentVolumeClaim @@ -43,7 +43,7 @@ spec: spec: containers: - name: pre-provisioned-pv-private-with-daemon-secret - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target diff --git a/test/integration/failed-manifests/rwo-pv.yaml b/test/integration/failed-manifests/rwo-pv.yaml index fe9412f1..3bb3fdd6 100644 --- a/test/integration/failed-manifests/rwo-pv.yaml +++ b/test/integration/failed-manifests/rwo-pv.yaml @@ -17,7 +17,7 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" --- apiVersion: v1 kind: PersistentVolumeClaim @@ -43,7 +43,7 @@ spec: spec: containers: - name: failed-rwo-pv - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -55,4 +55,4 @@ spec: - name: target persistentVolumeClaim: claimName: pvc-test-csi-image-rwo - backoffLimit: 0 \ No newline at end of file + backoffLimit: 0 diff --git a/test/integration/failed-manifests/rwx-pv.yaml b/test/integration/failed-manifests/rwx-pv.yaml index cf8298e2..308ac673 100644 --- a/test/integration/failed-manifests/rwx-pv.yaml +++ b/test/integration/failed-manifests/rwx-pv.yaml @@ -17,7 +17,7 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" --- apiVersion: v1 kind: PersistentVolumeClaim @@ -43,7 +43,7 @@ spec: spec: containers: - name: failed-rwx-pv - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -55,4 +55,4 @@ spec: - name: target persistentVolumeClaim: claimName: pvc-test-csi-image-rwx - backoffLimit: 0 \ No newline at end of file + backoffLimit: 0 diff --git a/test/integration/manifests/ephemeral-volume-docker-library-image.yaml b/test/integration/manifests/ephemeral-volume-docker-library-image.yaml index 57be52ac..6f605757 100644 --- a/test/integration/manifests/ephemeral-volume-docker-library-image.yaml +++ b/test/integration/manifests/ephemeral-volume-docker-library-image.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: ephemeral-volume-docker-library-image - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -22,6 +22,6 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "warmmetal/csi-image-test:simple-fs" + image: "warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" backoffLimit: 0 diff --git a/test/integration/manifests/ephemeral-volume-private-with-given-secret.yaml b/test/integration/manifests/ephemeral-volume-private-with-given-secret.yaml index 383d5364..def5c305 100644 --- a/test/integration/manifests/ephemeral-volume-private-with-given-secret.yaml +++ b/test/integration/manifests/ephemeral-volume-private-with-given-secret.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: ephemeral-volume-private-with-given-secret - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -24,6 +24,6 @@ spec: nodePublishSecretRef: name: warmmetal volumeAttributes: - image: "private-registry:5000/warmmetal/csi-image-test:simple-fs" + image: "private-registry:5000/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" backoffLimit: 0 diff --git a/test/integration/manifests/ephemeral-volume.yaml b/test/integration/manifests/ephemeral-volume.yaml index d2beeb4d..0188ed7a 100644 --- a/test/integration/manifests/ephemeral-volume.yaml +++ b/test/integration/manifests/ephemeral-volume.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: ephemeral-volume - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -22,5 +22,5 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" backoffLimit: 0 diff --git a/test/integration/manifests/pre-provisioned-pv-private-with-given-secret.yaml b/test/integration/manifests/pre-provisioned-pv-private-with-given-secret.yaml index d4cb830f..7771da2d 100644 --- a/test/integration/manifests/pre-provisioned-pv-private-with-given-secret.yaml +++ b/test/integration/manifests/pre-provisioned-pv-private-with-given-secret.yaml @@ -20,7 +20,7 @@ spec: nodePublishSecretRef: name: warmmetal namespace: default - volumeHandle: "private-registry:5000/warmmetal/csi-image-test:simple-fs" + volumeHandle: "private-registry:5000/warmmetal/container-image-csi-driver-test:simple-fs" volumeAttributes: pullAlways: "true" --- @@ -48,7 +48,7 @@ spec: spec: containers: - name: pre-provisioned-pv-private-with-given-secret - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target diff --git a/test/integration/manifests/pre-provisioned-pv.yaml b/test/integration/manifests/pre-provisioned-pv.yaml index 16c3e4da..73332c92 100644 --- a/test/integration/manifests/pre-provisioned-pv.yaml +++ b/test/integration/manifests/pre-provisioned-pv.yaml @@ -7,7 +7,7 @@ provisioner: csi-image.warm-metal.tech apiVersion: v1 kind: PersistentVolume metadata: - name: pv-test-csi-image-test-simple-fs + name: pv-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech capacity: @@ -17,14 +17,14 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" volumeAttributes: pullAlways: "true" --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: pvc-test-csi-image-test-simple-fs + name: pvc-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech accessModes: @@ -32,7 +32,7 @@ spec: resources: requests: storage: 5Gi - volumeName: pv-test-csi-image-test-simple-fs + volumeName: pv-test-container-image-csi-driver-test-simple-fs --- apiVersion: batch/v1 kind: Job @@ -45,7 +45,7 @@ spec: spec: containers: - name: pre-provisioned-pv - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -56,5 +56,5 @@ spec: volumes: - name: target persistentVolumeClaim: - claimName: pvc-test-csi-image-test-simple-fs + claimName: pvc-test-container-image-csi-driver-test-simple-fs backoffLimit: 0 diff --git a/test/integration/manifests/readonly-ephemeral-volume.yaml b/test/integration/manifests/readonly-ephemeral-volume.yaml index 94e91095..6fef9607 100644 --- a/test/integration/manifests/readonly-ephemeral-volume.yaml +++ b/test/integration/manifests/readonly-ephemeral-volume.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: readonly-ephemeral-volume - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -25,6 +25,6 @@ spec: driver: csi-image.warm-metal.tech readOnly: true volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" backoffLimit: 0 diff --git a/test/integration/manifests/readonly-mount-ephemeral-volume.yaml b/test/integration/manifests/readonly-mount-ephemeral-volume.yaml index 4c749897..d79abcb3 100644 --- a/test/integration/manifests/readonly-mount-ephemeral-volume.yaml +++ b/test/integration/manifests/readonly-mount-ephemeral-volume.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: readonly-mount-ephemeral-volume - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -25,6 +25,6 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" backoffLimit: 0 diff --git a/test/integration/manifests/readonly-mount-pre-pv.yaml b/test/integration/manifests/readonly-mount-pre-pv.yaml index f0f56e5d..d6587b6c 100644 --- a/test/integration/manifests/readonly-mount-pre-pv.yaml +++ b/test/integration/manifests/readonly-mount-pre-pv.yaml @@ -7,7 +7,7 @@ provisioner: csi-image.warm-metal.tech apiVersion: v1 kind: PersistentVolume metadata: - name: pv-test-csi-image-test-simple-fs + name: pv-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech capacity: @@ -17,14 +17,14 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" volumeAttributes: pullAlways: "true" --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: pvc-test-csi-image-test-simple-fs + name: pvc-test-container-image-csi-driver-test-simple-fs spec: storageClassName: csi-image.warm-metal.tech accessModes: @@ -32,7 +32,7 @@ spec: resources: requests: storage: 5Gi - volumeName: pv-test-csi-image-test-simple-fs + volumeName: pv-test-container-image-csi-driver-test-simple-fs --- apiVersion: batch/v1 kind: Job @@ -45,7 +45,7 @@ spec: spec: containers: - name: pre-provisioned-pv - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target @@ -59,5 +59,5 @@ spec: volumes: - name: target persistentVolumeClaim: - claimName: pvc-test-csi-image-test-simple-fs + claimName: pvc-test-container-image-csi-driver-test-simple-fs backoffLimit: 0 diff --git a/test/integration/manifests/two-readonly-ephemeral-volume.yaml b/test/integration/manifests/two-readonly-ephemeral-volume.yaml index 261481c4..355aad75 100644 --- a/test/integration/manifests/two-readonly-ephemeral-volume.yaml +++ b/test/integration/manifests/two-readonly-ephemeral-volume.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: two-readonly-ephemeral-volume - image: docker.io/warmmetal/csi-image-test:stat-fs + image: docker.io/warmmetal/container-image-csi-driver-test:stat-fs env: - name: TARGET1 value: /target1 @@ -27,13 +27,13 @@ spec: driver: csi-image.warm-metal.tech readOnly: true volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" - name: target2 csi: driver: csi-image.warm-metal.tech readOnly: true volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" pullAlways: "true" backoffLimit: 0 diff --git a/test/integration/manifests/two-readonly-pv.yaml b/test/integration/manifests/two-readonly-pv.yaml index 2dcec747..2be60207 100644 --- a/test/integration/manifests/two-readonly-pv.yaml +++ b/test/integration/manifests/two-readonly-pv.yaml @@ -7,7 +7,7 @@ provisioner: csi-image.warm-metal.tech apiVersion: v1 kind: PersistentVolume metadata: - name: pv-test-csi-image-test-simple-fs-1 + name: pv-test-container-image-csi-driver-test-simple-fs-1 spec: storageClassName: csi-image.warm-metal.tech capacity: @@ -17,14 +17,14 @@ spec: persistentVolumeReclaimPolicy: Retain csi: driver: csi-image.warm-metal.tech - volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" volumeAttributes: pullAlways: "true" --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: pvc-test-csi-image-test-simple-fs-1 + name: pvc-test-container-image-csi-driver-test-simple-fs-1 spec: storageClassName: csi-image.warm-metal.tech accessModes: @@ -32,12 +32,12 @@ spec: resources: requests: storage: 5Gi - volumeName: pv-test-csi-image-test-simple-fs-1 + volumeName: pv-test-container-image-csi-driver-test-simple-fs-1 --- # apiVersion: v1 # kind: PersistentVolume # metadata: -# name: pv-test-csi-image-test-simple-fs-2 +# name: pv-test-container-image-csi-driver-test-simple-fs-2 # spec: # storageClassName: csi-image.warm-metal.tech # capacity: @@ -47,14 +47,14 @@ spec: # persistentVolumeReclaimPolicy: Retain # csi: # driver: csi-image.warm-metal.tech -# volumeHandle: "docker.io/warmmetal/csi-image-test:simple-fs" +# volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" # volumeAttributes: # pullAlways: "true" --- # apiVersion: v1 # kind: PersistentVolumeClaim # metadata: -# name: pvc-test-csi-image-test-simple-fs-2 +# name: pvc-test-container-image-csi-driver-test-simple-fs-2 # spec: # storageClassName: csi-image.warm-metal.tech # accessModes: @@ -62,7 +62,7 @@ spec: # resources: # requests: # storage: 5Gi -# volumeName: pv-test-csi-image-test-simple-fs-2 +# volumeName: pv-test-container-image-csi-driver-test-simple-fs-2 --- apiVersion: batch/v1 kind: Job @@ -75,7 +75,7 @@ spec: spec: containers: - name: two-ro-pre-provisioned-pv - image: docker.io/warmmetal/csi-image-test:check-fs + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs env: - name: TARGET value: /target1 @@ -94,8 +94,8 @@ spec: volumes: - name: target1 persistentVolumeClaim: - claimName: pvc-test-csi-image-test-simple-fs-1 + claimName: pvc-test-container-image-csi-driver-test-simple-fs-1 # - name: target2 # persistentVolumeClaim: - # claimName: pvc-test-csi-image-test-simple-fs-2 + # claimName: pvc-test-container-image-csi-driver-test-simple-fs-2 backoffLimit: 0 diff --git a/test/integration/manifests/two-readonly-single-pv.yaml b/test/integration/manifests/two-readonly-single-pv.yaml new file mode 100644 index 00000000..ed9252a1 --- /dev/null +++ b/test/integration/manifests/two-readonly-single-pv.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-test-multi-read-simple-fs +spec: + storageClassName: csi-image.warm-metal.tech + capacity: + storage: 5Mi + accessModes: + - ReadOnlyMany # does not force volume to be mounted read only + persistentVolumeReclaimPolicy: Delete + csi: + driver: csi-image.warm-metal.tech + volumeHandle: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" + volumeAttributes: + pullAlways: "true" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pv-test-multi-read-simple-fs +spec: + storageClassName: csi-image.warm-metal.tech + accessModes: + - ReadOnlyMany + resources: + requests: + storage: 5Mi + volumeName: pv-test-multi-read-simple-fs +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: two-ro-single-pv-read-only-true +spec: + template: + metadata: + name: two-ro-single-pv-read-only-true + spec: + containers: + - name: two-ro-single-pv-read-only-true + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs + env: + - name: TARGET + value: /target1 + - name: CHECK_RO + value: "true" + volumeMounts: + - mountPath: /target1 + readOnly: true + name: target1 + restartPolicy: Never + volumes: + - name: target1 + persistentVolumeClaim: + claimName: pv-test-multi-read-simple-fs + backoffLimit: 0 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: two-ro-single-pv-read-only-false +spec: + template: + metadata: + name: two-ro-single-pv-read-only-false + spec: + containers: + - name: two-ro-single-pv-read-only-false + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs + env: + - name: TARGET + value: /target1 + - name: CHECK_RO + value: "true" + volumeMounts: + - mountPath: /target1 + name: target1 + restartPolicy: Never + volumes: + - name: target1 + persistentVolumeClaim: + claimName: pv-test-multi-read-simple-fs + backoffLimit: 0 diff --git a/test/integration/manual-manifests/write-check.yaml b/test/integration/manual-manifests/write-check.yaml index ca5de61d..cb2b2a79 100644 --- a/test/integration/manual-manifests/write-check.yaml +++ b/test/integration/manual-manifests/write-check.yaml @@ -7,10 +7,10 @@ spec: metadata: name: write-check spec: - nodeName: csi-image-test-m02 + nodeName: container-image-csi-driver-test-m02 containers: - name: write-check - image: docker.io/warmmetal/csi-image-test:write-check + image: docker.io/warmmetal/container-image-csi-driver-test:write-check env: - name: TARGET value: /target @@ -23,5 +23,5 @@ spec: csi: driver: csi-image.warm-metal.tech volumeAttributes: - image: "docker.io/warmmetal/csi-image-test:simple-fs" - backoffLimit: 2 \ No newline at end of file + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" + backoffLimit: 2 diff --git a/test/integration/metrics-manifests/error-ephemeral-volume.yaml b/test/integration/metrics-manifests/error-ephemeral-volume.yaml new file mode 100644 index 00000000..e425e63f --- /dev/null +++ b/test/integration/metrics-manifests/error-ephemeral-volume.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: error-ephemeral-volume +spec: + template: + metadata: + name: error-ephemeral-volume + spec: + containers: + - name: error-ephemeral-volume + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs + env: + - name: TARGET + value: /target + volumeMounts: + - mountPath: /target + name: target + restartPolicy: Never + volumes: + - name: target + csi: + driver: csi-image.warm-metal.tech + volumeAttributes: + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs-doesnt-exist" + # # set pullAlways if you want to ignore local images + # pullAlways: "true" + # # set secret if the image is private + # secret: "name of the ImagePullSecret" + # secretNamespace: "namespace of the secret" + backoffLimit: 0 diff --git a/test/integration/metrics-manifests/no-error-ephemeral-volume.yaml b/test/integration/metrics-manifests/no-error-ephemeral-volume.yaml new file mode 100644 index 00000000..0bc9cf6b --- /dev/null +++ b/test/integration/metrics-manifests/no-error-ephemeral-volume.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: no-error-compatible-ephemeral-volume +spec: + template: + metadata: + name: no-error-compatible-ephemeral-volume + spec: + containers: + - name: no-error-compatible-ephemeral-volume + image: docker.io/warmmetal/container-image-csi-driver-test:check-fs + env: + - name: TARGET + value: /target + volumeMounts: + - mountPath: /target + name: target + restartPolicy: Never + volumes: + - name: target + csi: + driver: csi-image.warm-metal.tech + volumeAttributes: + image: "docker.io/warmmetal/container-image-csi-driver-test:simple-fs" + # # set pullAlways if you want to ignore local images + # pullAlways: "true" + # # set secret if the image is private + # secret: "name of the ImagePullSecret" + # secretNamespace: "namespace of the secret" + backoffLimit: 0 diff --git a/test/integration/metrics-manifests/rendered-test.yaml b/test/integration/metrics-manifests/rendered-test.yaml new file mode 100644 index 00000000..03d3d446 --- /dev/null +++ b/test/integration/metrics-manifests/rendered-test.yaml @@ -0,0 +1,25 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-metrics +spec: + template: + metadata: + name: test-metrics + spec: + containers: + - name: test-metrics + # official curl iamge taken from https://github.com/curl/curl-container + image: quay.io/curl/curl:latest + command: + - /bin/sh + - -c + - | + (curl $IP:8080/metrics | grep warm_metal_pull_duration_seconds) && \ + (curl $IP:8080/metrics | grep warm_metal_mount_duration_seconds) && \ + (curl $IP:8080/metrics | grep warm_metal_operation_errors_total) + env: + - name: IP + value: 172.18.0.2 + restartPolicy: Never + backoffLimit: 0 diff --git a/test/integration/metrics-manifests/test.yaml b/test/integration/metrics-manifests/test.yaml new file mode 100644 index 00000000..75e90b01 --- /dev/null +++ b/test/integration/metrics-manifests/test.yaml @@ -0,0 +1,25 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-metrics +spec: + template: + metadata: + name: test-metrics + spec: + containers: + - name: test-metrics + # official curl iamge taken from https://github.com/curl/curl-container + image: quay.io/curl/curl:latest + command: + - /bin/sh + - -c + - | + (curl $IP:8080/metrics | grep warm_metal_pull_duration_seconds) && \ + (curl $IP:8080/metrics | grep warm_metal_mount_duration_seconds) && \ + (curl $IP:8080/metrics | grep warm_metal_operation_errors_total) + env: + - name: IP + value: %IP + restartPolicy: Never + backoffLimit: 0 diff --git a/test/integration/restart-runtime.sh b/test/integration/restart-runtime.sh index ccc6fdc6..fdd05bf3 100755 --- a/test/integration/restart-runtime.sh +++ b/test/integration/restart-runtime.sh @@ -12,10 +12,10 @@ PodRestartCount=$1 TestBase=$(dirname "${BASH_SOURCE[0]}") # Save kubelet, containerd and cri-o configuration for restarting -minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo mkdir -p /var/lib/boot2docker/etc/kubernetes -minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo cp -rpf /etc/kubernetes/* /var/lib/boot2docker/etc/kubernetes/ || true +minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo mkdir -p /var/lib/boot2docker/etc/kubernetes +minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo cp -rpf /etc/kubernetes/* /var/lib/boot2docker/etc/kubernetes/ || true # urcloud is a hacked minikube binary which can be downloaded from "". -urcloud cp -p csi-image-test "${TestBase}/minikube-bootlocal.sh" "csi-image-test-m02:/var/lib/boot2docker/bootlocal.sh" +urcloud cp -p container-image-csi-driver-test "${TestBase}/minikube-bootlocal.sh" "container-image-csi-driver-test-m02:/var/lib/boot2docker/bootlocal.sh" manifest=${TestBase}/manual-manifests/write-check.yaml kubectl delete --ignore-not-found -f "$manifest" @@ -47,23 +47,23 @@ for c in ${containers[@]}; do c=${c#"${CRIO_PREFIX}"} fi - containerPID=$(minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo crictl inspect -o go-template --template '{{.info.pid}}' ${c} | tr -d '\r\n' | awk '{$1=$1};1') - minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo kill -USR1 ${containerPID} + containerPID=$(minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo crictl inspect -o go-template --template '{{.info.pid}}' ${c} | tr -d '\r\n' | awk '{$1=$1};1') + minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo kill -USR1 ${containerPID} pidsToBeTerminated+=( ${containerPID} ) done echo "Wait 1sec for file writing" sleep 1 -minikube -p csi-image-test node stop csi-image-test-m02 -urcloud -p csi-image-test node start csi-image-test-m02 --rejoin=false --preload=false -minikube -p csi-image-test image load k8s.gcr.io/pause:3.2 -minikube -p csi-image-test image load k8s.gcr.io/kube-proxy:v1.20.2 +minikube -p container-image-csi-driver-test node stop container-image-csi-driver-test-m02 +urcloud -p container-image-csi-driver-test node start container-image-csi-driver-test-m02 --rejoin=false --preload=false +minikube -p container-image-csi-driver-test image load k8s.gcr.io/pause:3.2 +minikube -p container-image-csi-driver-test image load k8s.gcr.io/kube-proxy:v1.20.2 -nodeStatus=$(kubectl get no csi-image-test-m02 --no-headers | awk '{ print $2 }') +nodeStatus=$(kubectl get no container-image-csi-driver-test-m02 --no-headers | awk '{ print $2 }') while [ "$nodeStatus" != "Ready" ]; do sleep 1 - nodeStatus=$(kubectl get no csi-image-test-m02 --no-headers | awk '{ print $2 }') + nodeStatus=$(kubectl get no container-image-csi-driver-test-m02 --no-headers | awk '{ print $2 }') done echo "Wait 5secs for pod status updating" @@ -95,11 +95,11 @@ for c in ${containers[@]}; do c=${c#"${CRIO_PREFIX}"} fi - containerPID=$(minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo crictl inspect -o go-template --template '{{.info.pid}}' ${c} | tr -d '\r\n' | awk '{$1=$1};1') - minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo kill -USR2 ${containerPID} + containerPID=$(minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo crictl inspect -o go-template --template '{{.info.pid}}' ${c} | tr -d '\r\n' | awk '{$1=$1};1') + minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo kill -USR2 ${containerPID} done kubectl wait --timeout=5m --for=condition=complete -f "${manifest}" set +x -set +e \ No newline at end of file +set +e diff --git a/test/integration/test-backward-compatability.sh b/test/integration/test-backward-compatability.sh index ae5caf3d..3efde875 100755 --- a/test/integration/test-backward-compatability.sh +++ b/test/integration/test-backward-compatability.sh @@ -3,10 +3,10 @@ set -e source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/cluster.sh lib::start_cluster_containerd ${K8S_VERSION} -minikube cache reload -p csi-image-test +minikube cache reload -p container-image-csi-driver-test source $(dirname "${BASH_SOURCE[0]}")/backward-compatability.sh echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/test-in-kind-containerd.sh b/test/integration/test-in-kind-containerd.sh index 84df2237..e4ef6739 100755 --- a/test/integration/test-in-kind-containerd.sh +++ b/test/integration/test-in-kind-containerd.sh @@ -23,7 +23,7 @@ $(dirname "${BASH_SOURCE[0]}")/../../hack/helper/kind_bed.sh 'k8s' trap "docker rm -f kind-${GITHUB_RUN_ID}-control-plane" ERR EXIT INT TERM $(dirname "${BASH_SOURCE[0]}")/../../hack/ci/setup_private_registry.sh -trap "docker rm -f private-registry; docker rmi localhost:5000/warmmetal/csi-image-test:simple-fs" ERR EXIT INT TERM +trap "docker rm -f private-registry; docker rmi localhost:5000/warmmetal/container-image-csi-driver-test:simple-fs" ERR EXIT INT TERM helm install ${HELM_NAME} $(dirname "${BASH_SOURCE[0]}")/../../charts/warm-metal-csi-driver -n kube-system \ -f ${VALUE_FILE} --set csiPlugin.image.tag=${IMAGE_TAG} --wait --debug @@ -32,4 +32,4 @@ $(dirname "${BASH_SOURCE[0]}")/../../hack/ci/test.sh helm uninstall -n kube-system ${HELM_NAME} --wait -set +e \ No newline at end of file +set +e diff --git a/test/integration/test-in-kind-crio.sh b/test/integration/test-in-kind-crio.sh index 4f79d77f..b89711a2 100755 --- a/test/integration/test-in-kind-crio.sh +++ b/test/integration/test-in-kind-crio.sh @@ -4,7 +4,7 @@ set -e export IMAGE_TAG=v0.7.0 export GITHUB_RUN_ID=1 -export NODE_IMAGE=docker.io/warmmetal/kindest-node-crio:v1.25.2-csi-driver-image +export NODE_IMAGE=docker.io/warmmetal/kindest-node-crio:v1.25.2-container-image-csi-driver export VALUE_FILE=$(dirname "${BASH_SOURCE[0]}")/crio-helm-values.yaml HELM_NAME='wm-csi-integration-tests' @@ -15,7 +15,7 @@ $(dirname "${BASH_SOURCE[0]}")/../../hack/helper/kind_bed.sh 'k8s' trap "docker rm -f kind-${GITHUB_RUN_ID}-control-plane" ERR EXIT INT TERM $(dirname "${BASH_SOURCE[0]}")/../../hack/ci/setup_private_registry.sh -trap "docker rm -f private-registry; docker rmi localhost:5000/warmmetal/csi-image-test:simple-fs" ERR EXIT INT TERM +trap "docker rm -f private-registry; docker rmi localhost:5000/warmmetal/container-image-csi-driver-test:simple-fs" ERR EXIT INT TERM helm install ${HELM_NAME} $(dirname "${BASH_SOURCE[0]}")/../../charts/warm-metal-csi-driver -n kube-system \ -f ${VALUE_FILE} \ @@ -29,4 +29,4 @@ $(dirname "${BASH_SOURCE[0]}")/../../hack/ci/test.sh helm uninstall -n kube-system ${HELM_NAME} --wait -set +e \ No newline at end of file +set +e diff --git a/test/integration/test-in-minikube-containerd.sh b/test/integration/test-in-minikube-containerd.sh index ca22bf64..94996deb 100755 --- a/test/integration/test-in-minikube-containerd.sh +++ b/test/integration/test-in-minikube-containerd.sh @@ -7,6 +7,6 @@ source $(dirname "${BASH_SOURCE[0]}")/../../hack/helper/prepare_containerd_clust source $(dirname "${BASH_SOURCE[0]}")/cases.sh lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test +minikube delete -p container-image-csi-driver-test -set +e \ No newline at end of file +set +e diff --git a/test/integration/test-in-minikube-cri-o.sh b/test/integration/test-in-minikube-cri-o.sh index 6398672f..f1ec16a7 100755 --- a/test/integration/test-in-minikube-cri-o.sh +++ b/test/integration/test-in-minikube-cri-o.sh @@ -10,13 +10,13 @@ lib::install_driver echo "Install a private registry" lib::install_private_registry -minikube ssh -p csi-image-test -- sudo podman pull docker.io/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo podman image tag docker.io/warmmetal/csi-image-test:simple-fs localhost:31000/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo podman push localhost:31000/warmmetal/csi-image-test:simple-fs --tls-verify=false --creds warmmetal:warmmetal +minikube ssh -p container-image-csi-driver-test -- sudo podman pull docker.io/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo podman image tag docker.io/warmmetal/container-image-csi-driver-test:simple-fs localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo podman push localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs --tls-verify=false --creds warmmetal:warmmetal source $(dirname "${BASH_SOURCE[0]}")/cases.sh lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/test-in-minikube-docker.sh b/test/integration/test-in-minikube-docker.sh index 3bdcc039..f21d95d8 100755 --- a/test/integration/test-in-minikube-docker.sh +++ b/test/integration/test-in-minikube-docker.sh @@ -10,13 +10,13 @@ lib::install_driver echo "Install a private registry" lib::install_private_registry -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i pull docker.io/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i tag --force docker.io/warmmetal/csi-image-test:simple-fs localhost:31000/warmmetal/csi-image-test:simple-fs -minikube ssh -p csi-image-test -- sudo ctr -n k8s.io i push localhost:31000/warmmetal/csi-image-test:simple-fs --plain-http --user warmmetal:warmmetal +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i pull docker.io/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i tag --force docker.io/warmmetal/container-image-csi-driver-test:simple-fs localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs +minikube ssh -p container-image-csi-driver-test -- sudo ctr -n k8s.io i push localhost:31000/warmmetal/container-image-csi-driver-test:simple-fs --plain-http --user warmmetal:warmmetal source $(dirname "${BASH_SOURCE[0]}")/cases.sh lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/test-metrics.sh b/test/integration/test-metrics.sh new file mode 100755 index 00000000..24a2b921 --- /dev/null +++ b/test/integration/test-metrics.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/utils.sh + +set -e + +TestBase=$(dirname "${BASH_SOURCE[0]}") +kubectl apply -f "${TestBase}/metrics-manifests/error-ephemeral-volume.yaml" +lib::run_test_job "${TestBase}/metrics-manifests/no-error-ephemeral-volume.yaml" + + +ip="$(kubectl get po -n kube-system -l=component=nodeplugin -ojsonpath='{.items[*].status.podIP}')" +cat "${TestBase}/metrics-manifests/test.yaml" | sed "s|%IP|$ip|g" > "${TestBase}/metrics-manifests/rendered-test.yaml" + +lib::run_test_job "${TestBase}/metrics-manifests/rendered-test.yaml" + +kubectl delete -f "${TestBase}/metrics-manifests/error-ephemeral-volume.yaml" + +set +e \ No newline at end of file diff --git a/test/integration/test-restart-ds-containerd.sh b/test/integration/test-restart-ds-containerd.sh index 049e4e4b..39930dfa 100755 --- a/test/integration/test-restart-ds-containerd.sh +++ b/test/integration/test-restart-ds-containerd.sh @@ -3,12 +3,12 @@ set -e source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/cluster.sh lib::start_cluster_containerd ${K8S_VERSION} -minikube cache reload -p csi-image-test +minikube cache reload -p container-image-csi-driver-test lib::install_driver source $(dirname "${BASH_SOURCE[0]}")/restart-ds.sh lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/test-restart-ds-crio.sh b/test/integration/test-restart-ds-crio.sh index 101a8616..354c91b1 100755 --- a/test/integration/test-restart-ds-crio.sh +++ b/test/integration/test-restart-ds-crio.sh @@ -3,12 +3,12 @@ set -e source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/cluster.sh lib::start_cluster_crio ${K8S_VERSION} -minikube cache reload -p csi-image-test +minikube cache reload -p container-image-csi-driver-test lib::install_driver source $(dirname "${BASH_SOURCE[0]}")/restart-ds.sh lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/test-restart-runtime-containerd.sh b/test/integration/test-restart-runtime-containerd.sh index af5c4009..957e8a2c 100755 --- a/test/integration/test-restart-runtime-containerd.sh +++ b/test/integration/test-restart-runtime-containerd.sh @@ -3,16 +3,16 @@ set -e source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/cluster.sh lib::start_cluster_containerd ${K8S_VERSION} -minikube node -p csi-image-test add -minikube cache reload -p csi-image-test +minikube node -p container-image-csi-driver-test add +minikube cache reload -p container-image-csi-driver-test lib::install_driver -minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo mkdir -p /mnt/vda1/var/lib/boot2docker/etc/containerd -minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo cp -rp /etc/containerd/* /var/lib/boot2docker/etc/containerd/ +minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo mkdir -p /mnt/vda1/var/lib/boot2docker/etc/containerd +minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo cp -rp /etc/containerd/* /var/lib/boot2docker/etc/containerd/ source $(dirname "${BASH_SOURCE[0]}")/restart-runtime.sh 1 lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/test-restart-runtime-crio.sh b/test/integration/test-restart-runtime-crio.sh index 692b8dea..d1c32e51 100755 --- a/test/integration/test-restart-runtime-crio.sh +++ b/test/integration/test-restart-runtime-crio.sh @@ -4,16 +4,16 @@ set -e source $(dirname "${BASH_SOURCE[0]}")/../../hack/lib/cluster.sh lib::start_cluster_crio ${K8S_VERSION} -minikube -p csi-image-test node add -minikube cache reload -p csi-image-test +minikube -p container-image-csi-driver-test node add +minikube cache reload -p container-image-csi-driver-test lib::install_driver -minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo mkdir -p /mnt/vda1/var/lib/boot2docker/etc/crio -minikube ssh -p csi-image-test -n csi-image-test-m02 -- sudo cp -rp /etc/crio/* /var/lib/boot2docker/etc/crio/ +minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo mkdir -p /mnt/vda1/var/lib/boot2docker/etc/crio +minikube ssh -p container-image-csi-driver-test -n container-image-csi-driver-test-m02 -- sudo cp -rp /etc/crio/* /var/lib/boot2docker/etc/crio/ source $(dirname "${BASH_SOURCE[0]}")/restart-runtime.sh 0 lib::uninstall_driver echo "Destroying cluster" -minikube delete -p csi-image-test -set +e \ No newline at end of file +minikube delete -p container-image-csi-driver-test +set +e diff --git a/test/integration/node-server/Dockerfile.containerd b/test/node-server/Dockerfile.containerd similarity index 100% rename from test/integration/node-server/Dockerfile.containerd rename to test/node-server/Dockerfile.containerd diff --git a/test/integration/node-server/README.md b/test/node-server/README.md similarity index 59% rename from test/integration/node-server/README.md rename to test/node-server/README.md index 4e42121b..e0f3f908 100644 --- a/test/integration/node-server/README.md +++ b/test/node-server/README.md @@ -1,12 +1,12 @@ -## What is this? +## What is this? This directory contains two files: 1. `docker-compose.yaml`: used to mount workspace on image built using `Dockerfile.containerd` 2. `Dockerfile.containerd`: used to run containerd ## Why? -This is to test `cmd/plugin/node_server_test.go` and `pkg/remoteimage/pull_test.go` +This is to run container-image-csi-driver code in a container where containerd is running -## How to run the tests? +## How to use? Build the image ```shell $ docker build . -f Dockerfile.containerd -t : @@ -26,7 +26,7 @@ For example: $ docker-compose up [+] Building 0.0s (0/0) docker:desktop-linux [+] Running 1/0 - ✔ Container node-server-containerd-workspace-1 Recreated 0.0s + ✔ Container node-server-containerd-workspace-1 Recreated 0.0s Attaching to node-server-containerd-workspace-1 node-server-containerd-workspace-1 | time="2023-11-08T09:03:00Z" level=warning msg="containerd config version `1` has been deprecated and will be removed in containerd v2.0, please switch to version `2`, see https://github.com/containerd/containerd/blob/main/docs/PLUGINS.md#version-header" node-server-containerd-workspace-1 | time="2023-11-08T09:03:00.484125750Z" level=info msg="starting containerd" revision=61f9fd88f79f081d64d6fa3bb1a0dc71ec870523 version=1.6.24 @@ -51,33 +51,6 @@ $ docker exec -it bash ``` For example: ```shell -$ docker exec -it 7769b9e621f1 bash ~ +$ docker exec -it 7769b9e621f1 bash ~ root@7769b9e621f1:/go# ``` - -**To run `TestNodePublishVolume`**: - -`cd` into `/code/cmd/plugin` and run `go test -run 'TestNodePublishVolume'` (note that `TestNodePublishVolume` is a regex) -``` -root@7769b9e621f1:/go# cd /code/cmd/plugin -root@7769b9e621f1:/code/cmd/plugin# go test -run 'TestNodePublishVolume' -I1108 10:27:03.784182 19936 mounter.go:45] load 0 snapshots from runtime -I1108 10:27:03.787347 19936 server.go:108] Listening for connections on address: &net.UnixAddr{Name:"//csi/csi.sock", Net:"unix"} -... -I1108 10:27:13.222601 19936 node_server_test.go:94] server was stopped -I1108 10:27:13.225907 19936 mounter.go:45] load 0 snapshots from runtime -I1108 10:27:13.235697 19936 server.go:108] Listening for connections on address: &net.UnixAddr{Name:"//csi/csi.sock", Net:"unix"} -... -PASS -ok github.com/warm-metal/csi-driver-image/cmd/plugin 46.711s -``` - -**To test `TestPull`**: -`cd` into `/code/pkg/remoteimage` and run `go test -run 'TestPull'` (note that `TestPull` is a regex) -``` -root@cdf7ee254501:~# cd /code/pkg/remoteimage -root@cdf7ee254501:/code/pkg/remoteimage# go test -run 'TestPull' -PASS -ok github.com/warm-metal/csi-driver-image/pkg/remoteimage 2.247s -root@cdf7ee254501:/code/pkg/remoteimage# -``` \ No newline at end of file diff --git a/test/integration/node-server/docker-compose.yaml b/test/node-server/docker-compose.yaml similarity index 100% rename from test/integration/node-server/docker-compose.yaml rename to test/node-server/docker-compose.yaml diff --git a/test/sanity/Dockerfile b/test/sanity/Dockerfile index d49b7047..0d3e4195 100644 --- a/test/sanity/Dockerfile +++ b/test/sanity/Dockerfile @@ -1,10 +1,10 @@ FROM docker.io/library/golang:1.16 -WORKDIR /go/src/csi-driver-image +WORKDIR /go/src/container-image-csi-driver COPY go.mod go.sum ./ RUN go mod download COPY sanity_test.go . -CMD go test -v ./... -ginkgo.skip='.*\[Controller\s+Server\].*' \ No newline at end of file +CMD go test -v ./... -ginkgo.skip='.*\[Controller\s+Server\].*' diff --git a/test/sanity/manifest.yaml b/test/sanity/manifest.yaml index 0ee3f12d..29d3a862 100644 --- a/test/sanity/manifest.yaml +++ b/test/sanity/manifest.yaml @@ -1,17 +1,17 @@ apiVersion: batch/v1 kind: Job metadata: - name: csi-driver-image-sanity-test + name: container-image-csi-driver-sanity-test namespace: cliapp-system spec: template: metadata: - name: csi-driver-image-sanity-test + name: container-image-csi-driver-sanity-test spec: serviceAccountName: csi-image-warm-metal containers: - - name: csi-driver-image-sanity-test - image: local.test/csi-driver-image-test:sanity + - name: container-image-csi-driver-sanity-test + image: local.test/container-image-csi-driver-test:sanity volumeMounts: - mountPath: /csi name: socket-dir diff --git a/test/unit/manifest.yaml b/test/unit/manifest.yaml index 85da70ce..07f59550 100644 --- a/test/unit/manifest.yaml +++ b/test/unit/manifest.yaml @@ -1,17 +1,17 @@ apiVersion: batch/v1 kind: Job metadata: - name: csi-driver-image-test + name: container-image-csi-driver-test namespace: kube-system spec: template: metadata: - name: csi-driver-image-test + name: container-image-csi-driver-test spec: serviceAccountName: csi-image-warm-metal containers: - - name: csi-driver-image-test - image: warmmetal/csi-driver-image-test + - name: container-image-csi-driver-test + image: warmmetal/container-image-csi-driver-test env: - name: HTTP_PROXY value: http://192.168.64.1:1087 diff --git a/test/unit/skaffold.yaml b/test/unit/skaffold.yaml index 3b748817..ada165a5 100644 --- a/test/unit/skaffold.yaml +++ b/test/unit/skaffold.yaml @@ -7,7 +7,7 @@ build: - image: warmmetal/csi-image docker: dockerfile: Dockerfile - - image: warmmetal/csi-driver-image-test + - image: warmmetal/container-image-csi-driver-test docker: dockerfile: "test.dockerfile" # custom: