From c2f6154bb1f03f55bec9cd1c4df05c59fc3cbe8e Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 28 Feb 2025 17:41:19 +0000
Subject: [PATCH 01/18] Check Kayobe version playbook

---
 etc/kayobe/ansible/check-kayobe-version.yml   | 42 +++++++++++++++++++
 .../pre.d/check-kayobe-version.yml            |  1 +
 .../pre.d/check-kayobe-version.yml            |  1 +
 .../pre.d/check-kayobe-version.yml            |  1 +
 .../pre.d/check-kayobe-version.yml            |  1 +
 ...check-kayobe-version-f44d43c9c34d1b89.yaml |  6 +++
 6 files changed, 52 insertions(+)
 create mode 100644 etc/kayobe/ansible/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml
 create mode 100644 releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
new file mode 100644
index 000000000..e087be1f7
--- /dev/null
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -0,0 +1,42 @@
+---
+- name: Check kayobe version
+  hosts: localhost
+  gather_facts: false
+  vars:
+    requirements_path: "{{ kayobe_config_path }}/../../requirements.txt"
+  tasks:
+    - name: Get installed kayobe commit
+      ansible.builtin.shell:
+        cmd: pip freeze | grep kayobe | cut -d @ -f 3 -s
+      register: kayobe_git_commit
+
+    - name: Clone kayobe
+      ansible.builtin.git:
+        repo: https://github.com/stackhpc/kayobe.git
+        dest: /tmp/kayobe-git
+        version: stackhpc/{{ openstack_release }}
+
+    - name: Get tag from kayobe commit
+      ansible.builtin.command:
+        cmd: git describe --tags {{ kayobe_git_commit.stdout }}
+        chdir: /tmp/kayobe-git
+      register: kayobe_current_version
+
+    - name: Get latest Kayobe version
+      ansible.builtin.shell:
+        cmd: grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2
+      register: kayobe_latest_version
+
+    - name: Check installed kayobe version is the latest
+      ansible.builtin.assert:
+        that: "kayobe_latest_version.stdout in kayobe_current_version.stdout"
+        fail_msg: |
+          Kayobe must be updated to the latest version before continuing.
+
+          Current Kayobe version: {{ kayobe_current_version.stdout }}
+          Latest Kayobe version: {{ kayobe_latest_version.stdout }}
+
+          Recreate the Kayobe environment, or install the latest version
+          by running: pip install --force-reinstall -r {{ requirements_path }}
+        success_msg: |
+          Kayobe running at version: {{ kayobe_current_version.stdout }}
diff --git a/etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml b/releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml
new file mode 100644
index 000000000..b4e257a99
--- /dev/null
+++ b/releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Ansible playbook to check the installed Kayobe version against
+    the version defined in Kayobe configuration. This playbook will
+    run on Kayobe bootstrap, host and service operations.

From 41dcf361f899f7f6c221a5c9b7dbe3b78fb20a3f Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 3 Mar 2025 13:56:22 +0000
Subject: [PATCH 02/18] Fix pipefail

---
 etc/kayobe/ansible/check-kayobe-version.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index e087be1f7..69242eb22 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -7,7 +7,8 @@
   tasks:
     - name: Get installed kayobe commit
       ansible.builtin.shell:
-        cmd: pip freeze | grep kayobe | cut -d @ -f 3 -s
+        cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3 -s
+        executable: /usr/bin/bash
       register: kayobe_git_commit
 
     - name: Clone kayobe
@@ -24,7 +25,8 @@
 
     - name: Get latest Kayobe version
       ansible.builtin.shell:
-        cmd: grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2
+        cmd: set -o pipefail && grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2
+        executable: /usr/bin/bash
       register: kayobe_latest_version
 
     - name: Check installed kayobe version is the latest

From 0ecaf9fcda849c4294b2029dd8965277001315f4 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 3 Mar 2025 15:32:52 +0000
Subject: [PATCH 03/18] Ignore linter

---
 .ansible-lint-ignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore
index aa444047f..3869facce 100644
--- a/.ansible-lint-ignore
+++ b/.ansible-lint-ignore
@@ -5,3 +5,4 @@ etc/kayobe/ansible/vault-generate-internal-tls.yml fqcn[action-core]
 etc/kayobe/ansible/vault-generate-test-external-tls.yml fqcn[action-core]
 etc/kayobe/ansible/rabbitmq-reset.yml command-instead-of-module
 etc/kayobe/ansible/ubuntu-upgrade.yml syntax-check[missing-file]
+etc/kayobe/ansible/check-kayobe-version.yml command-instead-of-module

From 77158029e386204a61f36bd80f73a9e4c2f6afe5 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 3 Mar 2025 15:34:23 +0000
Subject: [PATCH 04/18] Include tag option

---
 etc/kayobe/ansible/check-kayobe-version.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index 69242eb22..23352388a 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -1,5 +1,6 @@
 ---
 - name: Check kayobe version
+  tags: kayobe-version-check
   hosts: localhost
   gather_facts: false
   vars:
@@ -7,7 +8,7 @@
   tasks:
     - name: Get installed kayobe commit
       ansible.builtin.shell:
-        cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3 -s
+        cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3
         executable: /usr/bin/bash
       register: kayobe_git_commit
 

From bcfbf89bc50b0653e83ed8102f6914e4e076565d Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 3 Mar 2025 16:03:51 +0000
Subject: [PATCH 05/18] Hook for overcloud host upgrade

---
 etc/kayobe/ansible/check-kayobe-version.yml            | 10 +++++-----
 .../pre.d/check-kayobe-version.yml                     |  1 +
 2 files changed, 6 insertions(+), 5 deletions(-)
 create mode 120000 etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index 23352388a..df98b6377 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -1,24 +1,24 @@
 ---
-- name: Check kayobe version
+- name: Check Kayobe version
   tags: kayobe-version-check
   hosts: localhost
   gather_facts: false
   vars:
     requirements_path: "{{ kayobe_config_path }}/../../requirements.txt"
   tasks:
-    - name: Get installed kayobe commit
+    - name: Get installed Kayobe commit
       ansible.builtin.shell:
         cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3
         executable: /usr/bin/bash
       register: kayobe_git_commit
 
-    - name: Clone kayobe
+    - name: Clone Kayobe
       ansible.builtin.git:
         repo: https://github.com/stackhpc/kayobe.git
         dest: /tmp/kayobe-git
         version: stackhpc/{{ openstack_release }}
 
-    - name: Get tag from kayobe commit
+    - name: Get tag from Kayobe commit
       ansible.builtin.command:
         cmd: git describe --tags {{ kayobe_git_commit.stdout }}
         chdir: /tmp/kayobe-git
@@ -30,7 +30,7 @@
         executable: /usr/bin/bash
       register: kayobe_latest_version
 
-    - name: Check installed kayobe version is the latest
+    - name: Check installed Kayobe version is the latest
       ansible.builtin.assert:
         that: "kayobe_latest_version.stdout in kayobe_current_version.stdout"
         fail_msg: |
diff --git a/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file

From 3169db84ca1707112ae570335bda597b7de159cb Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 3 Mar 2025 16:37:52 +0000
Subject: [PATCH 06/18] Check on control host upgrade

---
 .../hooks/control-host-upgrade/pre.d/check-kayobe-version.yml    | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml

diff --git a/etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file

From c30f7218bc2f42f3cf01cf790f79671c3748cf7d Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 4 Mar 2025 15:29:25 +0000
Subject: [PATCH 07/18] Include remaining hooks

---
 .../hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml | 1 +
 .../hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml | 1 +
 .../overcloud-service-reconfigure/pre.d/check-kayobe-version.yml | 1 +
 .../hooks/seed-host-configure/pre.d/check-kayobe-version.yml     | 1 +
 .../pre.d/check-kayobe-version.yml                               | 1 +
 .../hooks/seed-service-deploy/pre.d/check-kayobe-version.yml     | 1 +
 6 files changed, 6 insertions(+)
 create mode 120000 etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml
 create mode 120000 etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml

diff --git a/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml
new file mode 120000
index 000000000..bec8ed4c6
--- /dev/null
+++ b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kayobe-version.yml
\ No newline at end of file

From ceeacc343d191554c7b5794709304629c785fd17 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 4 Mar 2025 15:30:48 +0000
Subject: [PATCH 08/18] Clarify fail message for unexpected current versions

---
 etc/kayobe/ansible/check-kayobe-version.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index df98b6377..fd59feb9f 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -34,12 +34,12 @@
       ansible.builtin.assert:
         that: "kayobe_latest_version.stdout in kayobe_current_version.stdout"
         fail_msg: |
-          Kayobe must be updated to the latest version before continuing.
+          Kayobe must use the expected version before continuing.
 
           Current Kayobe version: {{ kayobe_current_version.stdout }}
-          Latest Kayobe version: {{ kayobe_latest_version.stdout }}
+          Expected Kayobe version: {{ kayobe_latest_version.stdout }}
 
-          Recreate the Kayobe environment, or install the latest version
+          Recreate the Kayobe environment, or install the expected version
           by running: pip install --force-reinstall -r {{ requirements_path }}
         success_msg: |
           Kayobe running at version: {{ kayobe_current_version.stdout }}

From 10d27b92c7ce563a65b85e83ffe176455c638688 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 5 Mar 2025 15:19:51 +0000
Subject: [PATCH 09/18] Kolla-Ansible version check playbook

---
 etc/kayobe/ansible/check-kolla-version.yml | 24 ++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 etc/kayobe/ansible/check-kolla-version.yml

diff --git a/etc/kayobe/ansible/check-kolla-version.yml b/etc/kayobe/ansible/check-kolla-version.yml
new file mode 100644
index 000000000..e8e7a7ec2
--- /dev/null
+++ b/etc/kayobe/ansible/check-kolla-version.yml
@@ -0,0 +1,24 @@
+---
+- name: Check Kolla-Ansible version
+  tags: kolla-version-check
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Get installed Kolla-Ansible commit
+      ansible.builtin.command:
+        cmd: git describe --tags
+        chdir: "{{ lookup('ansible.builtin.env', 'KOLLA_SOURCE_PATH') }}"
+      register: kolla_ansible_current_version
+
+    - name: Check installed Kolla-Ansible version is the latest
+      ansible.builtin.assert:
+        that: "stackhpc_kolla_ansible_source_version in kolla_ansible_current_version.stdout"
+        fail_msg: |
+          Kolla-Ansible must use the expected version before continuing.
+
+          Current Kolla-Ansible version: {{ kolla_ansible_current_version.stdout }}
+          Expected Kolla-Ansible version: {{ stackhpc_kolla_ansible_source_version }}
+
+          Upgrade Kolla-Ansible by running: kayobe control host upgrade
+        success_msg: |
+          Kolla-Ansible running at version: {{ kolla_ansible_current_version.stdout }}

From 88dba74a8d7621c6ce44b0326892ca79d23004ec Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 5 Mar 2025 16:50:34 +0000
Subject: [PATCH 10/18] Add pip 24 precheck

---
 etc/kayobe/ansible/check-kayobe-version.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index fd59feb9f..307c4a46a 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -6,6 +6,18 @@
   vars:
     requirements_path: "{{ kayobe_config_path }}/../../requirements.txt"
   tasks:
+    - name: Get package info
+      community.general.pip_package_info:
+      register: packages
+
+    - name: Check if pip is version 24.0 or newer
+      ansible.builtin.assert:
+        that: "{{ packages.packages.pip.pip[0].version is version('24.0', '>=') }}"
+        fail_msg: |
+          Pip must be 24.0 or newer to run this check. Upgrade pip by running
+          pip install -U pip and reinstall Kayobe by running:
+          pip install --force-reinstall -r {{ requirements_path }}
+
     - name: Get installed Kayobe commit
       ansible.builtin.shell:
         cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3

From c46cec92fe937cfaf3434ffda90360fd21835c6b Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 6 Mar 2025 12:13:00 +0000
Subject: [PATCH 11/18] Kolla-Ansible check hooks

---
 .../hooks/infra-vm-host-configure/pre.d/check-kolla-version.yml  | 1 +
 .../hooks/infra-vm-service-deploy/pre.d/check-kolla-version.yml  | 1 +
 .../hooks/overcloud-host-upgrade/pre.d/check-kolla-version.yml   | 1 +
 .../hooks/overcloud-service-deploy/pre.d/check-kolla-version.yml | 1 +
 .../overcloud-service-reconfigure/pre.d/check-kolla-version.yml  | 1 +
 .../hooks/seed-host-configure/pre.d/check-kolla-version.yml      | 1 +
 .../seed-hypervisor-host-configure/pre.d/check-kolla-version.yml | 1 +
 .../hooks/seed-service-deploy/pre.d/check-kolla-version.yml      | 1 +
 8 files changed, 8 insertions(+)
 create mode 120000 etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-version.yml
 create mode 120000 etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-version.yml

diff --git a/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file

From ef48b1db1a6311cc6cfb681890000e4d049f7749 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 6 Mar 2025 12:14:45 +0000
Subject: [PATCH 12/18] service upgrade hook

---
 .../overcloud-service-upgrade/pre.d/check-kolla-version.yml      | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-version.yml

diff --git a/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file

From 63b68485534358f3d99adcda8a217527a434abad Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 6 Mar 2025 12:15:48 +0000
Subject: [PATCH 13/18] s/latest/expected/g

---
 etc/kayobe/ansible/check-kolla-version.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/check-kolla-version.yml b/etc/kayobe/ansible/check-kolla-version.yml
index e8e7a7ec2..3db9b3285 100644
--- a/etc/kayobe/ansible/check-kolla-version.yml
+++ b/etc/kayobe/ansible/check-kolla-version.yml
@@ -4,13 +4,13 @@
   hosts: localhost
   gather_facts: false
   tasks:
-    - name: Get installed Kolla-Ansible commit
+    - name: Get current Kolla-Ansible tag
       ansible.builtin.command:
         cmd: git describe --tags
         chdir: "{{ lookup('ansible.builtin.env', 'KOLLA_SOURCE_PATH') }}"
       register: kolla_ansible_current_version
 
-    - name: Check installed Kolla-Ansible version is the latest
+    - name: Check installed Kolla-Ansible version is the expected version
       ansible.builtin.assert:
         that: "stackhpc_kolla_ansible_source_version in kolla_ansible_current_version.stdout"
         fail_msg: |

From dd8f122472e92a060318644c6a58a4be090886d7 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 6 Mar 2025 12:19:29 +0000
Subject: [PATCH 14/18] Fixup Reno

---
 .ansible-lint-ignore                                        | 1 +
 .../notes/check-kayobe-version-f44d43c9c34d1b89.yaml        | 6 ------
 .../notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml  | 6 ++++++
 3 files changed, 7 insertions(+), 6 deletions(-)
 delete mode 100644 releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml
 create mode 100644 releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml

diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore
index 3869facce..641ee891b 100644
--- a/.ansible-lint-ignore
+++ b/.ansible-lint-ignore
@@ -6,3 +6,4 @@ etc/kayobe/ansible/vault-generate-test-external-tls.yml fqcn[action-core]
 etc/kayobe/ansible/rabbitmq-reset.yml command-instead-of-module
 etc/kayobe/ansible/ubuntu-upgrade.yml syntax-check[missing-file]
 etc/kayobe/ansible/check-kayobe-version.yml command-instead-of-module
+etc/kayobe/ansible/check-kolla-version.yml command-instead-of-module
diff --git a/releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml b/releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml
deleted file mode 100644
index b4e257a99..000000000
--- a/releasenotes/notes/check-kayobe-version-f44d43c9c34d1b89.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-features:
-  - |
-    Ansible playbook to check the installed Kayobe version against
-    the version defined in Kayobe configuration. This playbook will
-    run on Kayobe bootstrap, host and service operations.
diff --git a/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml b/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml
new file mode 100644
index 000000000..9de7c2c86
--- /dev/null
+++ b/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Ansible playbooks to check the installed Kayobe/Kolla-Ansible versions
+    with the expected versions in Kayobe configuration. These checks 
+    will run on Kayobe bootstrap, host and service operations.

From 7c534cc10c9f5580a1c78518a548de7da2ed453b Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 6 Mar 2025 12:20:58 +0000
Subject: [PATCH 15/18] Add hook for host configure

---
 .../hooks/overcloud-host-configure/pre.d/check-kolla-version.yml | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-version.yml

diff --git a/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-version.yml b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-version.yml
new file mode 120000
index 000000000..482d85cac
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-version.yml
@@ -0,0 +1 @@
+../../../ansible/check-kolla-version.yml
\ No newline at end of file

From d1136a0f6f99445f0ec024f7f52bfbf402f04a88 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 7 Mar 2025 16:00:48 +0000
Subject: [PATCH 16/18] Include option to skip in config

---
 etc/kayobe/ansible/check-kayobe-version.yml   | 101 +++++++++---------
 etc/kayobe/ansible/check-kolla-version.yml    |  33 +++---
 etc/kayobe/stackhpc.yml                       |   8 ++
 ...kayobe-version-check-f44d43c9c34d1b89.yaml |   6 ++
 4 files changed, 84 insertions(+), 64 deletions(-)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index 307c4a46a..c2f0bcdac 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -6,52 +6,55 @@
   vars:
     requirements_path: "{{ kayobe_config_path }}/../../requirements.txt"
   tasks:
-    - name: Get package info
-      community.general.pip_package_info:
-      register: packages
-
-    - name: Check if pip is version 24.0 or newer
-      ansible.builtin.assert:
-        that: "{{ packages.packages.pip.pip[0].version is version('24.0', '>=') }}"
-        fail_msg: |
-          Pip must be 24.0 or newer to run this check. Upgrade pip by running
-          pip install -U pip and reinstall Kayobe by running:
-          pip install --force-reinstall -r {{ requirements_path }}
-
-    - name: Get installed Kayobe commit
-      ansible.builtin.shell:
-        cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3
-        executable: /usr/bin/bash
-      register: kayobe_git_commit
-
-    - name: Clone Kayobe
-      ansible.builtin.git:
-        repo: https://github.com/stackhpc/kayobe.git
-        dest: /tmp/kayobe-git
-        version: stackhpc/{{ openstack_release }}
-
-    - name: Get tag from Kayobe commit
-      ansible.builtin.command:
-        cmd: git describe --tags {{ kayobe_git_commit.stdout }}
-        chdir: /tmp/kayobe-git
-      register: kayobe_current_version
-
-    - name: Get latest Kayobe version
-      ansible.builtin.shell:
-        cmd: set -o pipefail && grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2
-        executable: /usr/bin/bash
-      register: kayobe_latest_version
-
-    - name: Check installed Kayobe version is the latest
-      ansible.builtin.assert:
-        that: "kayobe_latest_version.stdout in kayobe_current_version.stdout"
-        fail_msg: |
-          Kayobe must use the expected version before continuing.
-
-          Current Kayobe version: {{ kayobe_current_version.stdout }}
-          Expected Kayobe version: {{ kayobe_latest_version.stdout }}
-
-          Recreate the Kayobe environment, or install the expected version
-          by running: pip install --force-reinstall -r {{ requirements_path }}
-        success_msg: |
-          Kayobe running at version: {{ kayobe_current_version.stdout }}
+    - name: Check version
+      block:
+        - name: Get package info
+          community.general.pip_package_info:
+          register: packages
+
+        - name: Check if pip is version 24.0 or newer
+          ansible.builtin.assert:
+            that: "{{ packages.packages.pip.pip[0].version is version('24.0', '>=') }}"
+            fail_msg: |
+              Pip must be 24.0 or newer to run this check. Upgrade pip by running
+              pip install -U pip and reinstall Kayobe by running:
+              pip install --force-reinstall -r {{ requirements_path }}
+
+        - name: Get installed Kayobe commit
+          ansible.builtin.shell:
+            cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3
+            executable: /usr/bin/bash
+          register: kayobe_git_commit
+
+        - name: Clone Kayobe
+          ansible.builtin.git:
+            repo: https://github.com/stackhpc/kayobe.git
+            dest: /tmp/kayobe-git
+            version: stackhpc/{{ openstack_release }}
+
+        - name: Get tag from Kayobe commit
+          ansible.builtin.command:
+            cmd: git describe --tags {{ kayobe_git_commit.stdout }}
+            chdir: /tmp/kayobe-git
+          register: kayobe_current_version
+
+        - name: Get latest Kayobe version
+          ansible.builtin.shell:
+            cmd: set -o pipefail && grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2
+            executable: /usr/bin/bash
+          register: kayobe_latest_version
+
+        - name: Check installed Kayobe version is the latest
+          ansible.builtin.assert:
+            that: "kayobe_latest_version.stdout in kayobe_current_version.stdout"
+            fail_msg: |
+              Kayobe must use the expected version before continuing.
+
+              Current Kayobe version: {{ kayobe_current_version.stdout }}
+              Expected Kayobe version: {{ kayobe_latest_version.stdout }}
+
+              Recreate the Kayobe environment, or install the expected version
+              by running: pip install --force-reinstall -r {{ requirements_path }}
+            success_msg: |
+              Kayobe running at version: {{ kayobe_current_version.stdout }}
+      when: stackhpc_enable_kayobe_check
diff --git a/etc/kayobe/ansible/check-kolla-version.yml b/etc/kayobe/ansible/check-kolla-version.yml
index 3db9b3285..9b19e6559 100644
--- a/etc/kayobe/ansible/check-kolla-version.yml
+++ b/etc/kayobe/ansible/check-kolla-version.yml
@@ -4,21 +4,24 @@
   hosts: localhost
   gather_facts: false
   tasks:
-    - name: Get current Kolla-Ansible tag
-      ansible.builtin.command:
-        cmd: git describe --tags
-        chdir: "{{ lookup('ansible.builtin.env', 'KOLLA_SOURCE_PATH') }}"
-      register: kolla_ansible_current_version
+    - name: Check version
+      block:
+        - name: Get current Kolla-Ansible tag
+          ansible.builtin.command:
+            cmd: git describe --tags
+            chdir: "{{ lookup('ansible.builtin.env', 'KOLLA_SOURCE_PATH') }}"
+          register: kolla_ansible_current_version
 
-    - name: Check installed Kolla-Ansible version is the expected version
-      ansible.builtin.assert:
-        that: "stackhpc_kolla_ansible_source_version in kolla_ansible_current_version.stdout"
-        fail_msg: |
-          Kolla-Ansible must use the expected version before continuing.
+        - name: Check installed Kolla-Ansible version is the expected version
+          ansible.builtin.assert:
+            that: "stackhpc_kolla_ansible_source_version in kolla_ansible_current_version.stdout"
+            fail_msg: |
+              Kolla-Ansible must use the expected version before continuing.
 
-          Current Kolla-Ansible version: {{ kolla_ansible_current_version.stdout }}
-          Expected Kolla-Ansible version: {{ stackhpc_kolla_ansible_source_version }}
+              Current Kolla-Ansible version: {{ kolla_ansible_current_version.stdout }}
+              Expected Kolla-Ansible version: {{ stackhpc_kolla_ansible_source_version }}
 
-          Upgrade Kolla-Ansible by running: kayobe control host upgrade
-        success_msg: |
-          Kolla-Ansible running at version: {{ kolla_ansible_current_version.stdout }}
+              Upgrade Kolla-Ansible by running: kayobe control host upgrade
+            success_msg: |
+              Kolla-Ansible running at version: {{ kolla_ansible_current_version.stdout }}
+      when: stackhpc_enable_kolla_check
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index bcfcdcacd..d64218fa3 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -175,6 +175,14 @@ stackhpc_docker_registry_password: "{{ pulp_password }}"
 # Whether or not to run CIS benchmark hardening playbooks. Default is false.
 #stackhpc_enable_cis_benchmark_hardening_hook:
 
+# Whether or not to check the installed Kayobe version when running Kayobe commands.
+# Default is true.
+stackhpc_enable_kayobe_check: true
+
+# Whether or not to check the installed Kolla-Ansible version when running Kayobe commands.
+# Default is true.
+stackhpc_enable_kolla_check: true
+
 ###############################################################################
 # Octavia Amphora image
 
diff --git a/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml b/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml
index 9de7c2c86..100e9f948 100644
--- a/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml
+++ b/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml
@@ -4,3 +4,9 @@ features:
     Ansible playbooks to check the installed Kayobe/Kolla-Ansible versions
     with the expected versions in Kayobe configuration. These checks 
     will run on Kayobe bootstrap, host and service operations.
+upgrade:
+  - |
+    Checks are enabled by default which may break existing deployments using
+    custom forks or branches for Kayobe and Kolla-Ansible, to disable version
+    checks in configuration set ``stackhpc_enable_kayobe_check`` and
+    ``stackhpc_enable_kolla_check`` to false

From 9e8980c59fe2b23ed122f15a832ea577a730089d Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 7 Mar 2025 16:05:49 +0000
Subject: [PATCH 17/18] Make linter happy

---
 etc/kayobe/ansible/check-kayobe-version.yml | 2 +-
 etc/kayobe/ansible/check-kolla-version.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index c2f0bcdac..2e4acf705 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -7,6 +7,7 @@
     requirements_path: "{{ kayobe_config_path }}/../../requirements.txt"
   tasks:
     - name: Check version
+      when: stackhpc_enable_kayobe_check
       block:
         - name: Get package info
           community.general.pip_package_info:
@@ -57,4 +58,3 @@
               by running: pip install --force-reinstall -r {{ requirements_path }}
             success_msg: |
               Kayobe running at version: {{ kayobe_current_version.stdout }}
-      when: stackhpc_enable_kayobe_check
diff --git a/etc/kayobe/ansible/check-kolla-version.yml b/etc/kayobe/ansible/check-kolla-version.yml
index 9b19e6559..74dc39741 100644
--- a/etc/kayobe/ansible/check-kolla-version.yml
+++ b/etc/kayobe/ansible/check-kolla-version.yml
@@ -5,6 +5,7 @@
   gather_facts: false
   tasks:
     - name: Check version
+      when: stackhpc_enable_kolla_check
       block:
         - name: Get current Kolla-Ansible tag
           ansible.builtin.command:
@@ -24,4 +25,3 @@
               Upgrade Kolla-Ansible by running: kayobe control host upgrade
             success_msg: |
               Kolla-Ansible running at version: {{ kolla_ansible_current_version.stdout }}
-      when: stackhpc_enable_kolla_check

From 24a005ee2cb42d4eb9f9153ea21e27336b9e867b Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 7 Mar 2025 16:34:56 +0000
Subject: [PATCH 18/18] Ensure commit hash is not empty

---
 etc/kayobe/ansible/check-kayobe-version.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml
index 2e4acf705..cc00b6aed 100644
--- a/etc/kayobe/ansible/check-kayobe-version.yml
+++ b/etc/kayobe/ansible/check-kayobe-version.yml
@@ -26,6 +26,7 @@
             cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3
             executable: /usr/bin/bash
           register: kayobe_git_commit
+          failed_when: kayobe_git_commit.stdout == ""
 
         - name: Clone Kayobe
           ansible.builtin.git: