Implement key rotation for ceph-fs

This patchset implements key rotation for the ceph-fs charm by
receiving the new pending key from the ceph-mon charm and
manually rotating it via Ceph's authtool. It makes use of the
'ceph-mds-relation-changed' hook for this.

Change-Id: I773f389f56d78cd7ce58f9f2b5e7d7695164acb1
func-test-pr: openstack-charmers/zaza-openstack-tests#1202

osci.yaml

@@ -2,7 +2,6 @@
     templates:
       - charm-unit-jobs-py38
       - charm-unit-jobs-py39
-      - charm-xena-functional-jobs
       - charm-yoga-functional-jobs
       - charm-functional-jobs
     vars:

src/reactive/ceph_fs.py

@@ -1,4 +1,4 @@
-# Copyright 2016 Canonical Ltd
+# Copyright 2024 Canonical Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,9 @@
 import charms_openstack.bus
 import charms_openstack.charm as charm
 
+import os
+import subprocess
+
 
 charms_openstack.bus.discover()
 
@@ -41,6 +44,9 @@
 def config_changed():
     ceph_mds = reactive.endpoint_from_flag('ceph-mds.pools.available')
     with charm.provide_charm_instance() as cephfs_charm:
+        host = cephfs_charm.hostname
+        exists = os.path.exists('/var/lib/ceph/mds/ceph-%s/keyring' % host)
+
         cephfs_charm.configure_ceph_keyring(ceph_mds.mds_key())
         cephfs_charm.render_with_interfaces([ceph_mds])
         if reactive.is_flag_set('config.changed.source'):
@@ -52,6 +58,22 @@ def config_changed():
         reactive.set_flag('config.rendered')
         cephfs_charm.assess_status()
 
+        # If the keyring file existed before this call, then the new
+        # provided key implies a rotation.
+        if exists:
+            svc = 'ceph-mds@%s.service' % host
+            try:
+                # Reset the failure count first, as the service may fail
+                # to come up due to the way the restart-map is handled.
+                subprocess.check_call(['sudo', 'systemctl',
+                                       'reset-failed', svc])
+                subprocess.check_call(['sudo', 'systemctl', 'restart', svc])
+            except subprocess.CalledProcessError as exc:
+                # The service can be temporarily masked when booting, so
+                # skip that class of errors.
+                ch_core.hookenv.log('Failed to restart MDS service: %s' %
+                                    str(exc))
+
 
 @reactive.when('ceph-mds.connected')
 def storage_ceph_connected(ceph):

src/tests/tests.yaml

@@ -17,6 +17,7 @@ tests:
   - zaza.openstack.charm_tests.ceph.fs.tests.CephFSTests
   - zaza.openstack.charm_tests.ceph.fs.tests.CharmOperationTest
   - zaza.openstack.charm_tests.ceph.tests.BlueStoreCompressionCharmOperation
+  - zaza.openstack.charm_tests.ceph.tests.CephMonKeyRotationTests
 
 target_deploy_status:
   ubuntu:

unit_tests/test_reactive_ceph_fs.py

@@ -61,6 +61,8 @@ def test_config_changed(self):
         self.patch_object(handlers.reactive, 'is_flag_set')
         self.patch_object(handlers.reactive, 'clear_flag')
         self.patch_object(handlers.reactive, 'set_flag')
+        self.patch_object(handlers.os.path, 'exists')
+        handlers.os.path.exists.return_value = False
         ceph_mds = mock.MagicMock()
         ceph_mds.mds_key.return_value = 'fakekey'
         self.endpoint_from_flag.return_value = ceph_mds