From d9af34ac232e0dcb022298074ed6b91c7e4476a9 Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Tue, 9 Apr 2024 11:32:46 +0900 Subject: [PATCH] Fixing broken mysql-router configuration A customer faced an issue when they face full disk. Mysql router configuration file was broken. As this charm doesn't handle whole configuration file, charm wrote just part of them based on template. so re-bootstrapping is needed when this symptom happens. To do that, I put code to check configuration file size, and if it is under 1000bytes, run bootstrap. func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1218 Closes-Bug: #2004088 Change-Id: Ic28f0406eb8d5dbd9131ae83090498297ac48c31 (cherry picked from commit d334665b2d8bb113223508f348d053020e8c860c) --- src/lib/charm/openstack/mysql_router.py | 30 +++++++++- src/reactive/mysql_router_handlers.py | 9 ++- .../test_lib_charm_openstack_mysql_router.py | 56 +++++++++++++++++++ unit_tests/test_mysql_router_handlers.py | 6 +- 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/src/lib/charm/openstack/mysql_router.py b/src/lib/charm/openstack/mysql_router.py index 6853189..658470d 100644 --- a/src/lib/charm/openstack/mysql_router.py +++ b/src/lib/charm/openstack/mysql_router.py @@ -479,7 +479,20 @@ def custom_assess_status_check(self): return None, None - def bootstrap_mysqlrouter(self): + def validate_configuration(self): + """Validate Configuration + + Check if mysql router configuration file is less than 1024 bytes. + If so, then the configuration file is probably damaged, and then + re-run the `bootstrap_mysqlrouter()` function with True to force + it. + """ + + conf_size = os.path.getsize(self.mysqlrouter_conf) + if conf_size <= 1024: + self.bootstrap_mysqlrouter(True) + + def bootstrap_mysqlrouter(self, force=False): """Bootstrap MySQL Router. Execute the mysqlrouter bootstrap command. MySQL Router bootstraps into @@ -494,7 +507,7 @@ def bootstrap_mysqlrouter(self): :rtype: None """ - if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED): + if not force and reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED): ch_core.hookenv.log( "Bootstrap mysqlrouter is being called after we set the " "bootstrapped flag: {}. This may require manual intervention," @@ -522,8 +535,19 @@ def bootstrap_mysqlrouter(self): # If we have attempted to bootstrap before but unsuccessfully, # use the force option to avoid LP Bug#1919560 - if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED): + is_bootstrap_attempted = reactive.flags.is_flag_set( + MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) + if is_bootstrap_attempted or force: cmd.append("--force") + # clear configuration before force bootstrap + # because if there are some regex string, it fails + try: + with open(self.mysqlrouter_conf, "wt") as f: + f.write("[DEFAULT]\n") + except Exception: + ch_core.hookenv.log( + "ignored, because the bootstrap will overwrite the file.") + pass # Set and attempt the bootstrap reactive.flags.set_flag(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) diff --git a/src/reactive/mysql_router_handlers.py b/src/reactive/mysql_router_handlers.py index af0e6ca..83820b0 100644 --- a/src/reactive/mysql_router_handlers.py +++ b/src/reactive/mysql_router_handlers.py @@ -11,7 +11,6 @@ charm.use_defaults( 'charm.installed', 'config.changed', - 'update-status', 'upgrade-charm') @@ -99,6 +98,7 @@ def proxy_shared_db_responses(shared_db, db_router): :type db_router_interface: MySQLRouterRequires object """ with charm.provide_charm_instance() as instance: + instance.validate_configuration() instance.config_changed() instance.proxy_db_and_user_responses(db_router, shared_db) instance.assess_status() @@ -112,3 +112,10 @@ def stop_charm(): with charm.provide_charm_instance() as instance: instance.stop_mysqlrouter() instance.config_cleanup() + + +@reactive.hook('update-status') +def update_status(): + with charm.provide_charm_instance() as instance: + instance.validate_configuration() + instance.assess_status() diff --git a/unit_tests/test_lib_charm_openstack_mysql_router.py b/unit_tests/test_lib_charm_openstack_mysql_router.py index d0cde5f..3647763 100644 --- a/unit_tests/test_lib_charm_openstack_mysql_router.py +++ b/unit_tests/test_lib_charm_openstack_mysql_router.py @@ -484,6 +484,62 @@ def test_bootstrap_mysqlrouter(self): self.clear_flag.assert_called_once_with( mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) + def test_bootstrap_mysqlrouter_force(self): + _json_addr = '"10.10.10.60"' + _json_pass = '"clusterpass"' + _pass = json.loads(_json_pass) + _addr = json.loads(_json_addr) + _user = "mysql" + _port = "3006" + self.patch_object(mysql_router.reactive.flags, "is_flag_set") + self.endpoint_from_flag.return_value = self.db_router + self.db_router.password.return_value = _json_pass + self.db_router.db_host.return_value = _json_addr + self.is_flag_set.return_value = False + + mrc = mysql_router.MySQLRouterCharm() + mrc.options.system_user = _user + mrc.options.base_port = _port + + _relations = ["relid"] + + self.patch_object(mysql_router.ch_core.hookenv, "relation_ids") + self.relation_ids.return_value = _relations + + _related_units = ["relunits"] + + self.patch_object(mysql_router.ch_core.hookenv, "related_units") + self.related_units.return_value = _related_units + + _config_data = { + "mysqlrouter_password": json.dumps(_pass), + "db_host": json.dumps(_addr), + } + + self.patch_object(mysql_router.ch_core.hookenv, "relation_get") + self.relation_get.return_value = _config_data + + self.cmp_pkgrevno.return_value = 1 + self.is_flag_set.side_effect = [False, True] + self.subprocess.check_output.side_effect = None + mrc.bootstrap_mysqlrouter(True) + self.subprocess.check_output.assert_called_once_with( + [mrc.mysqlrouter_bin, "--user", _user, "--name", mrc.name, + "--bootstrap", "{}:{}@{}" + .format(mrc.db_router_user, _pass, _addr), + "--directory", mrc.mysqlrouter_working_dir, + "--conf-use-sockets", + "--conf-bind-address", mrc.shared_db_address, + "--report-host", mrc.db_router_address, + "--conf-base-port", _port, + "--disable-rest", "--force"], + stderr=self.stdout) + self.set_flag.assert_has_calls([ + mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED), + mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAPPED)]) + self.clear_flag.assert_called_once_with( + mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) + def test_start_mysqlrouter(self): self.patch_object(mysql_router.ch_core.host, "service_start") _name = "keystone-mysql-router" diff --git a/unit_tests/test_mysql_router_handlers.py b/unit_tests/test_mysql_router_handlers.py index 705451c..b65a814 100644 --- a/unit_tests/test_mysql_router_handlers.py +++ b/unit_tests/test_mysql_router_handlers.py @@ -25,7 +25,6 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks): def test_hooks(self): defaults = [ "config.changed", - "update-status", "upgrade-charm", "charm.installed", ] @@ -56,7 +55,10 @@ def test_hooks(self): "hook": { "stop_charm": ( "stop", - ) + ), + "update_status": ( + "update-status", + ), } } # test that the hooks were registered via the