[bobcat] ObjectRetrierWraps fixes (cherry-pick #1234)

requirements.txt

@@ -50,7 +50,10 @@ python-novaclient
 python-octaviaclient
 python-swiftclient
 python-watcherclient
-tenacity
+# Due to https://github.com/jd/tenacity/pull/479 the strategy for mocking out tenacity
+# waits/times/etc no longer works.  Pin to 8.4.1 until it is solved.
+# Bug in tenacity tracking issue: https://github.com/jd/tenacity/issues/482
+tenacity<8.4.2
 paramiko
 
 # Documentation requirements

unit_tests/utilities/test_utilities.py

@@ -113,6 +113,28 @@ def func(self):
 
         mock_sleep.assert_not_called()
 
+    @mock.patch("time.sleep")
+    def test_object_wrap_multilevel_with_exception(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        class B:
+
+            def __init__(self):
+                self.a = A()
+
+        b = B()
+        # retry on a specific exception
+        wrapped_b = utilities.ObjectRetrierWraps(
+            b, num_retries=1, retry_exceptions=[SomeException])
+        with self.assertRaises(SomeException):
+            wrapped_b.a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
     @mock.patch("time.sleep")
     def test_log_called(self, mock_sleep):
 
@@ -128,9 +150,7 @@ def func(self):
         with self.assertRaises(SomeException):
             wrapped_a.func()
 
-        # there should be two calls; one for the single retry and one for the
-        # failure.
-        self.assertEqual(mock_log.call_count, 2)
+        mock_log.assert_called()
 
     @mock.patch("time.sleep")
     def test_back_off_maximum(self, mock_sleep):

unit_tests/utilities/test_zaza_utilities_openstack.py

@@ -99,6 +99,9 @@ def setUp(self):
         self.neutronclient.list_agents.return_value = self.agents
         self.neutronclient.list_bgp_speaker_on_dragent.return_value = \
             self.bgp_speakers
+        self.patch("zaza.openstack.utilities.ObjectRetrierWraps",
+                   name="_object_retrier_wraps",
+                   new=lambda x, *_, **__: x)
 
     def test_create_port(self):
         self.patch_object(openstack_utils, "get_net_uuid")

zaza/openstack/charm_tests/audit/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Keystone audit middleware.
+
+Collection of code for setting up and testing Keystone audit middleware
+functionality.
+"""

zaza/openstack/charm_tests/audit/tests.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+#
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Keystone audit middleware API logging testing.
+
+These methods test the rendering of the charm api-paste.ini file to
+ensure the appropriate sections are rendered or not rendered depending
+on the state of the audit-middleware configuration option.
+"""
+
+import textwrap
+import logging
+import zaza.model
+import zaza.openstack.charm_tests.test_utils as test_utils
+
+
+class KeystoneAuditMiddlewareTest(test_utils.OpenStackBaseTest):
+    """Keystone audit middleware test class."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for Keystone audit middleware tests."""
+        super(KeystoneAuditMiddlewareTest, cls).setUpClass()
+        test_config = cls.test_config['tests_options']['audit-middleware']
+        cls.service_name = test_config['service']
+
+        cls.application_name = test_config.get('application', cls.service_name)
+        logging.info('Using application name: %s', cls.application_name)
+
+        cls.initial_audit_middleware = zaza.model.get_application_config(
+            cls.application_name)['audit-middleware']['value']
+
+    @classmethod
+    def tearDownClass(cls):
+        """Restore the audit-middleware configuration to its original state."""
+        super(KeystoneAuditMiddlewareTest, cls).tearDownClass()
+        logging.info("Running teardown on %s" % cls.application_name)
+        zaza.model.set_application_config(
+            cls.application_name,
+            {'audit-middleware': str(cls.initial_audit_middleware)},
+            model_name=cls.model_name
+        )
+        zaza.model.wait_for_application_states(
+            states={cls.application_name: {
+                'workload-status': 'active',
+                'workload-status-message': 'Unit is ready'}},
+            model_name=cls.model_name
+        )
+
+    def fetch_api_paste_content(self):
+        """Fetch content of api-paste.ini file."""
+        api_paste_ini_path = f"/etc/{self.service_name}/api-paste.ini"
+        lead_unit = zaza.model.get_lead_unit_name(
+            self.application_name,
+            model_name=self.model_name
+        )
+        try:
+            return zaza.model.file_contents(
+                lead_unit,
+                api_paste_ini_path,
+            )
+        except zaza.model.CommandRunFailed as e:
+            self.fail("Error fetching api-paste.ini: %s" % e)
+
+    def test_101_apipaste_includes_audit_section(self):
+        """Test api-paste.ini renders audit section when enabled."""
+        expected_content = textwrap.dedent(f"""\
+            [filter:audit]
+            paste.filter_factory = keystonemiddleware.audit:filter_factory
+            audit_map_file = /etc/{self.service_name}/api_audit_map.conf
+            service_name = {self.service_name}
+            """)
+
+        set_default = {'audit-middleware': False}
+        set_alternate = {'audit-middleware': True}
+
+        with self.config_change(default_config=set_default,
+                                alternate_config=set_alternate,
+                                application_name=self.application_name):
+            api_paste_content = self.fetch_api_paste_content()
+            self.assertIn(expected_content, api_paste_content)
+
+    def test_102_apipaste_excludes_audit_section(self):
+        """Test api_paste.ini does not render audit section when disabled."""
+        section_heading = '[filter:audit]'
+        set_default = {'audit-middleware': True}
+        set_alternate = {'audit-middleware': False}
+
+        with self.config_change(default_config=set_default,
+                                alternate_config=set_alternate,
+                                application_name=self.application_name):
+            api_paste_content = self.fetch_api_paste_content()
+            self.assertNotIn(section_heading, api_paste_content)
+
+
+class IronicAuditMiddlewareTest(KeystoneAuditMiddlewareTest):
+    """Ironic-API audit middleware test class."""
+
+    def test_101_apipaste_includes_audit_section(self):
+        """Test api-paste.ini renders audit section when enabled."""
+        self.skipTest('ironic-api does not use an api-paste.ini file')
+
+    def test_102_apipaste_excludes_audit_section(self):
+        """Test api_paste.ini does not render audit section when disabled."""
+        self.skipTest('ironic-api does not use an api-paste.ini file')

zaza/openstack/charm_tests/cinder_backup/tests.py

@@ -22,6 +22,7 @@
 
 import zaza.model
 import zaza.openstack.charm_tests.test_utils as test_utils
+from zaza.openstack.utilities import retry_on_connect_failure
 import zaza.openstack.utilities.ceph as ceph_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 
@@ -35,8 +36,9 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
     def setUpClass(cls):
         """Run class setup for running Cinder Backup tests."""
         super(CinderBackupTest, cls).setUpClass()
-        cls.cinder_client = openstack_utils.get_cinder_session_client(
-            cls.keystone_session)
+        cls.cinder_client = retry_on_connect_failure(
+            openstack_utils.get_cinder_session_client(cls.keystone_session),
+            log=logging.warn)
 
     @property
     def services(self):
@@ -101,7 +103,7 @@ def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self):
                     self.cinder_client.volumes,
                     cinder_vol.id,
                     wait_iteration_max_time=180,
-                    stop_after_attempt=15,
+                    stop_after_attempt=30,
                     expected_status='available',
                     msg='ceph-backed cinder volume')
 
@@ -123,7 +125,7 @@ def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self):
                     self.cinder_client.backups,
                     vol_backup.id,
                     wait_iteration_max_time=180,
-                    stop_after_attempt=15,
+                    stop_after_attempt=30,
                     expected_status='available',
                     msg='Backup volume')
 

zaza/openstack/charm_tests/designate/tests.py

@@ -134,6 +134,12 @@ def wait():
 
     def test_300_default_soa_config_options(self):
         """Configure default SOA options."""
+        current_release = openstack_utils.get_os_release()
+        jammy_antelope = openstack_utils.get_os_release('jammy_antelope')
+        if current_release > jammy_antelope:
+            self.skipTest('changing default ttl is currently broken since '
+                          'jammy_bobcat due to LP#2042944')
+
         test_domain = "test_300_example.com."
         DEFAULT_TTL = 60
         alternate_config = {'default-soa-minimum': 600,

zaza/openstack/charm_tests/manila/tests.py

@@ -359,7 +359,7 @@ def test_manila_share(self):
             self.manila_client.shares,
             share.id,
             wait_iteration_max_time=120,
-            stop_after_attempt=2,
+            stop_after_attempt=10,
             expected_status="available",
             msg="Waiting for a share to become available")
 

zaza/openstack/charm_tests/mysql/tests.py

@@ -1145,3 +1145,66 @@ def test_910_restart_on_config_change(self):
             {}, {},
             self.services)
         logging.info("Passed restart on changed test.")
+
+    def test_920_mysqlrouter_conf_broken(self):
+        """Checking conf broken case.
+
+        Run the bootstrap when conf is broken
+        """
+        # application_name on test is keystone-mysql-router
+        # using self.conf_file introduces error.
+        # instead of changing current self.conf_file,
+        # define one (it could introduce another issue)
+        config_file = (
+            "/var/lib/mysql/{}/mysqlrouter.conf"
+            .format(self.application_name))
+
+        logging.info("Starting broken conf test")
+
+        # put empty string to conf_file and make it wrong
+        logging.info("Breaking configuration file")
+        zaza.model.run_on_leader(self.application,
+                                 "echo '[DEFAULT]\n \
+                                        [metadata_cache:[\\w$]+$] \
+                                        ' > {}".format(
+                                     config_file))
+
+        logging.info("Getting configuration file")
+        recovered = zaza.model.run_on_leader(self.application,
+                                             "cat {}".format(
+                                                 config_file))['Stdout']
+
+        # Checking conf file length,
+        # if file is broken it is around 250
+        assert len(recovered) < 1000, (
+            "Breaking mysqlrouter conf failed.")
+
+        # verify it is in error state
+        for attempt in tenacity.Retrying(
+            reraise=True,
+            wait=tenacity.wait_fixed(10),
+            stop=tenacity.stop_after_attempt(30),
+        ):
+            with attempt:
+                # update status to make the status error
+                logging.info("Run update-status")
+                self.run_update_status_hooks(['keystone-mysql-router/0'])
+
+                # get current status
+                unit_status = (zaza.model.get_status()
+                               .applications
+                               ['keystone-mysql-router']['status'])
+                logging.info("Status:{}".format(unit_status['status']))
+                self.assertEqual(unit_status['status'], "active")
+
+        logging.info("Getting configuration file")
+        recovered = zaza.model.run_on_leader(self.application,
+                                             "cat {}".format(
+                                                 config_file))['Stdout']
+
+        # Checking conf file length,
+        # if file is broken it is around 250
+        assert len(recovered) > 1000, (
+            "Fixing mysqlrouter conf failed.")
+
+        logging.info("Passed broken conf test.")