feat(vmware): Support network events

This patch updates the DataSource for VMware to support network
reconfiguration when the BOOT, BOOT_LEGACY, BOOT_NEW_INSTANCE,
and HOTPLUG events are received.

Previously the datasource could only reconfigure the network if
a new instance ID was detected. However, due to features like
backup/restore, migrating VMs, etc., it was determined that it is
valuable to support reconfiguring the network without changing the
instance ID. This is because changing the instance ID also means
running the per-instance configuration modules, such as regenerating
the system's SSH host keys, which could lock out automation.

This patch also indicates the datasource supports network reconfig
for HOTPLUG events, such as when a new NIC is added to a VM.

cloudinit/sources/DataSourceVMware.py

@@ -26,8 +26,10 @@
 import os
 import socket
 import time
+from typing import Dict, Set
 
 from cloudinit import atomic_helper, dmi, net, netinfo, sources, util
+from cloudinit.event import EventScope, EventType, userdata_to_events
 from cloudinit.log import loggers
 from cloudinit.sources.helpers.vmware.imc import guestcust_util
 from cloudinit.subp import ProcessExecutionError, subp, which
@@ -53,6 +55,20 @@
 WAIT_ON_NETWORK_IPV4 = "ipv4"
 WAIT_ON_NETWORK_IPV6 = "ipv6"
 
+SUPPORTED_UPDATE_EVENTS = {
+    # Support network reconfiguration for the following events:
+    # TODO(akutz) Add support for METADATA_CHANGE and USER_REQUEST when
+    #             those events are supported by Cloud-Init.
+    EventScope.NETWORK: {
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+        EventType.BOOT_NEW_INSTANCE,
+        # EventType.HOTPLUG, # Add when subtype != IMC
+    }
+}
+
+SUPPORTED_UPDATE_EVENTS_GUEST_INFO_KEY = "cloudinit.supported-update-events"
+
 
 class DataSourceVMware(sources.DataSource):
     """
@@ -100,6 +116,8 @@ def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.data_access_method = None
         self.rpctool = None
         self.rpctool_fn = None
+        self.default_update_events = None
+        self.supported_update_events = copy.deepcopy(SUPPORTED_UPDATE_EVENTS)
 
         # A list includes all possible data transports, each tuple represents
         # one data transport type. This datasource will try to get data from
@@ -121,6 +139,18 @@ def _unpickle(self, ci_pkl_version: int) -> None:
                 setattr(self, attr, None)
         if not hasattr(self, "cfg"):
             setattr(self, "cfg", {})
+        if not hasattr(self, "supported_update_events"):
+            setattr(
+                self,
+                "supported_update_events",
+                copy.deepcopy(SUPPORTED_UPDATE_EVENTS),
+            )
+        if not hasattr(self, "default_update_events"):
+            setattr(
+                self,
+                "default_update_events",
+                copy.deepcopy(self.supported_update_events),
+            )
         if not hasattr(self, "possible_data_access_method_list"):
             setattr(
                 self,
@@ -182,6 +212,19 @@ def _get_data(self):
 
         LOG.info("using data access method %s", self._get_subplatform())
 
+        # Support HOTPLUG events when the data access method is something
+        # other than IMC.
+        if self.data_access_method != DATA_ACCESS_METHOD_IMC:
+            self.supported_update_events[EventScope.NETWORK].add(
+                EventType.HOTPLUG
+            )
+
+        # The default list of configured update events should be the same as
+        # the list of supported update events.
+        self.default_update_events = copy.deepcopy(
+            self.supported_update_events
+        )
+
         # Get the metadata.
         self.metadata = process_metadata(load_json_or_yaml(md))
 
@@ -229,6 +272,22 @@ def setup(self, is_new_instance):
         # persisted with the metadata.
         self.persist_instance_data()
 
+    def activate(self, cfg, is_new_instance):
+        """activate(cfg, is_new_instance)
+
+        This is called before the init_modules will be called but after
+        the user-data and vendor-data have been fully processed.
+
+        The cfg is fully up to date config, it contains a merged view of
+           system config, datasource config, user config, vendor config.
+        It should be used rather than the sys_cfg passed to __init__.
+
+        is_new_instance is a boolean indicating if this is a new instance.
+        """
+
+        # Reflect the supported update events into guestinfo.
+        self.advertise_supported_events(cfg)
+
     def _get_subplatform(self):
         get_key_name_fn = None
         if self.data_access_method == DATA_ACCESS_METHOD_ENVVAR:
@@ -262,6 +321,23 @@ def network_config(self):
             }
         return self.metadata["network"]["config"]
 
+    def advertise_supported_events(self, cfg):
+        default_events: Dict[EventScope, Set[EventType]] = copy.deepcopy(
+            self.default_update_events
+        )
+        user_events: Dict[EventScope, Set[EventType]] = userdata_to_events(
+            cfg.get("updates", {})
+        )
+        allowed_events = util.mergemanydict(
+            [
+                copy.deepcopy(user_events),
+                copy.deepcopy(default_events),
+            ]
+        )
+        return advertise_supported_events(
+            allowed_events, self.rpctool, self.rpctool_fn
+        )
+
     def get_instance_id(self):
         # Pull the instance ID out of the metadata if present. Otherwise
         # read the file /sys/class/dmi/id/product_uuid for the instance ID.
@@ -527,6 +603,40 @@ def advertise_local_ip_addrs(host_info, rpctool, rpctool_fn):
         LOG.info("advertised local ipv6 address %s in guestinfo", local_ipv6)
 
 
+def advertise_supported_events(supported_update_events, rpctool, rpctool_fn):
+    """
+    advertise_supported_events publishes the types of supported events
+    to guestinfo
+    """
+    if not rpctool or not rpctool_fn:
+        return
+
+    sz = ""
+    for event_scope in supported_update_events:
+        if len(sz) > 0:
+            sz += ","
+        sz += "{}=".format(event_scope)
+        event_types = []
+        for event_type in supported_update_events[event_scope]:
+            event_types.append("{}".format(event_type))
+        event_types.sort()
+        szt = ""
+        for event_type in event_types:
+            if len(szt) > 0:
+                szt += ";"
+            szt += event_type
+        sz += szt
+
+    if len(sz) > 0:
+        guestinfo_set_value(
+            SUPPORTED_UPDATE_EVENTS_GUEST_INFO_KEY, sz, rpctool, rpctool_fn
+        )
+        LOG.info("advertised supported update events in guestinfo: %s", sz)
+        return sz
+
+    return None
+
+
 def handle_returned_guestinfo_val(key, val):
     """
     handle_returned_guestinfo_val returns the provided value if it is

doc/rtd/reference/datasources/vmware.rst

@@ -325,6 +325,30 @@ If either of the above values are true, then the datasource will sleep for a
 second, check the network status, and repeat until one or both addresses from
 the specified families are available.
 
+Update Event support
+--------------------
+
+The VMware datasource supports the following types of update events:
+
+* Network -- `boot`, `boot-legacy`, and `boot-new-instance`
+
+The network event type `hotplug` is also supported for all transports except
+IMC.
+
+Determining the supported update events
+---------------------------------------
+
+This datasource also advertises the scope and type of the supported events.
+The ``guestinfo`` key ``guestinfo.cloudinit.supported-update-events`` contains
+ a list of the supported scopes and types that adheres to the format
+``SCOPE=TYPE[;TYPE][,SCOPE=TYPE[;TYPE]]``, for example:
+
+* ``network=boot;hotplug``
+* ``network=boot-new-instance``
+
+The advertised value is based on both the datasource's default set of
+supported events and those that may have been provided via user data.
+
 Walkthrough of GuestInfo keys transport
 =======================================
 

tests/unittests/sources/test_vmware.py

@@ -14,6 +14,7 @@
 import pytest
 
 from cloudinit import dmi, helpers, safeyaml, settings, util
+from cloudinit.event import EventScope, EventType
 from cloudinit.sources import DataSourceVMware
 from cloudinit.sources.helpers.vmware.imc import guestcust_util
 from cloudinit.subp import ProcessExecutionError
@@ -106,6 +107,19 @@
     "addr": "fd42:baa2:3dd:17a:216:3eff:fe16:db54",
 }
 
+VMW_EXPECTED_DEFAULT_EVENTS = {
+    EventType.BOOT,
+    EventType.BOOT_LEGACY,
+    EventType.BOOT_NEW_INSTANCE,
+    EventType.HOTPLUG,
+}
+
+VMW_EXPECTED_DEFAULT_EVENTS_IMC = {
+    EventType.BOOT,
+    EventType.BOOT_LEGACY,
+    EventType.BOOT_NEW_INSTANCE,
+}
+
 
 def generate_test_netdev_data(ipv4=None, ipv6=None):
     ipv4 = ipv4 or []
@@ -414,6 +428,14 @@ def test_wait_on_network(self, m_fn):
         self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4])
         self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1")
 
+    @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_set_value")
+    def test_advertise_supported_events(self, m_set_fn):
+        sz = DataSourceVMware.advertise_supported_events(
+            DataSourceVMware.SUPPORTED_UPDATE_EVENTS, "rpctool", len
+        )
+        self.assertEqual(1, m_set_fn.call_count)
+        self.assertEqual("network=boot;boot-legacy;boot-new-instance", sz)
+
 
 class TestDataSourceVMwareEnvVars(FilesystemMockingTestCase):
     """
@@ -469,6 +491,16 @@ def test_get_subplatform(self, m_fn):
             ),
         )
 
+        # Test to ensure that network is configured from metadata on each boot.
+        self.assertSetEqual(
+            VMW_EXPECTED_DEFAULT_EVENTS,
+            ds.default_update_events[EventScope.NETWORK],
+        )
+        self.assertSetEqual(
+            VMW_EXPECTED_DEFAULT_EVENTS,
+            ds.supported_update_events[EventScope.NETWORK],
+        )
+
     @mock.patch(
         "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
     )
@@ -602,6 +634,16 @@ def test_get_subplatform(self, m_which_fn, m_fn):
             ),
         )
 
+        # Test to ensure that network is configured from metadata on each boot.
+        self.assertSetEqual(
+            VMW_EXPECTED_DEFAULT_EVENTS,
+            ds.default_update_events[EventScope.NETWORK],
+        )
+        self.assertSetEqual(
+            VMW_EXPECTED_DEFAULT_EVENTS,
+            ds.supported_update_events[EventScope.NETWORK],
+        )
+
     @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
     @mock.patch("cloudinit.sources.DataSourceVMware.which")
     def test_get_data_metadata_with_vmware_rpctool(self, m_which_fn, m_fn):
@@ -702,6 +744,40 @@ def test_get_data_metadata_gz_b64(self, m_which_fn, m_fn):
         m_fn.side_effect = [data, "gz+b64", "", ""]
         self.assert_get_data_ok(m_fn, m_fn_call_count=4)
 
+    @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_set_value")
+    @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
+    @mock.patch("cloudinit.sources.DataSourceVMware.which")
+    def test_advertise_supported_events(self, m_which_fn, m_get_fn, m_set_fn):
+        m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"]
+        m_get_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""]
+        ds = self.assert_get_data_ok(m_get_fn, m_fn_call_count=4)
+        sz = ds.advertise_supported_events({})
+        self.assertEqual(1, m_set_fn.call_count)
+        self.assertEqual(
+            "network=boot;boot-legacy;boot-new-instance;hotplug", sz
+        )
+
+    @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_set_value")
+    @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
+    @mock.patch("cloudinit.sources.DataSourceVMware.which")
+    def test_advertise_supported_events_with_events_from_user_data(
+        self, m_which_fn, m_get_fn, m_set_fn
+    ):
+        m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"]
+        m_get_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""]
+        ds = self.assert_get_data_ok(m_get_fn, m_fn_call_count=4)
+        sz = ds.advertise_supported_events(
+            {
+                "updates": {
+                    "network": {
+                        "when": ["boot"],
+                    },
+                },
+            }
+        )
+        self.assertEqual(1, m_set_fn.call_count)
+        self.assertEqual("network=boot", sz)
+
 
 class TestDataSourceVMwareGuestInfo_InvalidPlatform(FilesystemMockingTestCase):
     """
@@ -746,6 +822,81 @@ def setUp(self):
         self.datasource = DataSourceVMware.DataSourceVMware
         self.tdir = self.tmp_dir()
 
+    def test_get_subplatform(self):
+        paths = helpers.Paths({"cloud_dir": self.tdir})
+        ds = self.datasource(
+            sys_cfg={"disable_vmware_customization": True},
+            distro={},
+            paths=paths,
+        )
+        # Prepare the conf file
+        conf_file = self.tmp_path("test-cust", self.tdir)
+        conf_content = dedent(
+            """\
+            [CLOUDINIT]
+            METADATA = test-meta
+            """
+        )
+        util.write_file(conf_file, conf_content)
+        # Prepare the meta data file
+        metadata_file = self.tmp_path("test-meta", self.tdir)
+        metadata_content = dedent(
+            """\
+            {
+              "instance-id": "cloud-vm",
+              "local-hostname": "my-host.domain.com",
+              "network": {
+                "version": 2,
+                "ethernets": {
+                  "eths": {
+                    "match": {
+                      "name": "ens*"
+                    },
+                    "dhcp4": true
+                  }
+                }
+              }
+            }
+            """
+        )
+        util.write_file(metadata_file, metadata_content)
+
+        with mock.patch(
+            MPATH + "guestcust_util.set_customization_status",
+            return_value=("msg", b""),
+        ):
+            result = wrap_and_call(
+                "cloudinit.sources.DataSourceVMware",
+                {
+                    "dmi.read_dmi_data": "vmware",
+                    "util.del_dir": True,
+                    "guestcust_util.search_file": self.tdir,
+                    "guestcust_util.wait_for_cust_cfg_file": conf_file,
+                    "guestcust_util.get_imc_dir_path": self.tdir,
+                },
+                ds._get_data,
+            )
+            self.assertTrue(result)
+
+        self.assertEqual(
+            ds.subplatform,
+            "%s (%s)"
+            % (
+                DataSourceVMware.DATA_ACCESS_METHOD_IMC,
+                DataSourceVMware.get_imc_key_name("metadata"),
+            ),
+        )
+
+        # Test to ensure that network is configured from metadata on each boot.
+        self.assertSetEqual(
+            VMW_EXPECTED_DEFAULT_EVENTS_IMC,
+            ds.default_update_events[EventScope.NETWORK],
+        )
+        self.assertSetEqual(
+            VMW_EXPECTED_DEFAULT_EVENTS_IMC,
+            ds.supported_update_events[EventScope.NETWORK],
+        )
+
     def test_get_data_false_on_none_dmi_data(self):
         """When dmi for system-product-name is None, get_data returns False."""
         paths = helpers.Paths({"cloud_dir": self.tdir})