Add support for set up virtual multi-asic chassis (sonic-net#16716)

What is the motivation for this PR?
Currently, the virtual chassis setup in sonic-mgmt is very primitive and manual.

How did you do it?
Enhanced the ansible framework to support automatic topo-connect and minigraph generation of virtual multi-asic chassis

How did you verify/test it?
Set up the virtual chassis with 202205/202405-chassis VS images

Any platform specific information?
Only tested to work with 202205 and 202405 images so far.

Author: BYGX-wcr

ansible/config_sonic_basedon_testbed.yml

@@ -93,12 +93,19 @@
     delegate_to: localhost
     ignore_errors: true
 
+  - name: determine whether to sort port_alias by index
+    set_fact:
+      sort_by_index: false
+    when: switch_type is defined and switch_type == "voq" and type is defined and type == "kvm"
+
   - name: find interface name mapping and individual interface speed if defined from dut
     port_alias:
       hwsku: "{{ hwsku }}"
       card_type: "{{ card_type | default('fixed') }}"
       hostname: "{{ inventory_hostname | default('') }}"
       switchids: "{{ switchids | default([]) }}"
+      num_asic: "{{ num_asics }}"
+      sort_by_index: "{{ sort_by_index | default(true) }}"
     when: deploy is defined and deploy|bool == true
 
   - name: find interface name mapping and individual interface speed if defined with local data
@@ -109,6 +116,7 @@
       hostname: "{{ inventory_hostname | default('') }}"
       switchids: "{{ switchids | default([]) }}"
       slotid: "{{ slot_num | default(None) }}"
+      sort_by_index: "{{ sort_by_index | default(true) }}"
     delegate_to: localhost
     when: deploy is not defined or deploy|bool == false
 

ansible/lab

@@ -161,6 +161,40 @@ sonic_a7260:
         "0x2d": "Arista Networks"
         "0x2e": "Aboot-norcal7-7.2.3-pcie2x4-12345678"
 
+sonic_nokia_multi_asic_lc:
+  vars:
+    hwsku: Nokia-IXR7250E-36x400G
+    iface_speed: 400000
+    num_asics: 2
+    start_topo_service: True
+    frontend_asics: [0,1]
+    card_type: linecard
+  hosts:
+    vlab-t2-03:
+      ansible_host: 10.250.0.123
+      ansible_hostv6: fec0::ffff:afa:13
+      slot_num: slot1
+      loopback4096_ip: [192.0.0.0/32, 192.0.0.1/32]
+      loopback4096_ipv6: [2603:10e2:400::/128, 2603:10e2:400::1/128]
+    vlab-t2-04:
+      ansible_host: 10.250.0.124
+      ansible_hostv6: fec0::ffff:afa:14
+      slot_num: slot2
+      loopback4096_ip: [192.0.0.3/32, 192.0.0.4/32]
+      loopback4096_ipv6: [2603:10e2:400::3/128, 2603:10e2:400::4/128]
+
+sonic_nokia_sup:
+  vars:
+    hwsku: Nokia-IXR7250E-SUP-10
+    iface_speed: 400000
+    start_topo_service: True
+    card_type: supervisor
+  hosts:
+    vlab-t2-sup1:
+      ansible_host: 10.250.0.125
+      ansible_hostv6: fec0::ffff:afa:15
+      slot_num: slot0
+
 sonic_multi_asic:
   vars:
     hwsku: msft_multi_asic_vs

ansible/library/port_alias.py

@@ -99,8 +99,10 @@ def get_portconfig_path(self, slotid=None, asic_id=None):
         return None
 
     def get_portmap(self, asic_id=None, include_internal=False,
-                    hostname=None, switchid=None, slotid=None):
+                    hostname=None, switchid=None, slotid=None, card_type=None):
         aliases = []
+        front_panel_aliases = []
+        inband_aliases = []
         portmap = {}
         aliasmap = {}
         portspeed = {}
@@ -109,7 +111,8 @@ def get_portmap(self, asic_id=None, include_internal=False,
         front_panel_asic_ifnames = {}
         front_panel_asic_id = {}
         # All asic names
-        asic_if_names = []
+        asic_if_names = {}
+        asic_if_ids = {}
         sysports = []
         port_coreid_index = -1
         port_core_portid_index = -1
@@ -171,22 +174,27 @@ def get_portmap(self, asic_id=None, include_internal=False,
                     else:
                         alias = name
                     add_port = False
+
+                    if role == "Ext":
+                        front_panel_aliases.append(alias)
+                    if role == "Inb":
+                        inband_aliases.append(alias)
+
                     if role in {"Ext"} or (role in ["Int", "Inb", "Rec"] and include_internal):
                         add_port = True
                         aliases.append(
                             (alias, -1 if port_index == -1 or len(mapping) <= port_index else mapping[port_index]))
                         portmap[name] = alias
                         aliasmap[alias] = name
-                        if role == "Ext" and (asic_name_index != -1) and (len(mapping) > asic_name_index):
+
+                        if (asic_name_index != -1) and (len(mapping) > asic_name_index):
                             asicifname = mapping[asic_name_index]
-                            # we only want following ASIC info in minigraph for multi-asic
                             if asic_id is not None:
-                                front_panel_asic_ifnames[alias] = asicifname
-                                front_panel_asic_id[alias] = "ASIC" + \
-                                    str(asic_id)
-                    if (asic_name_index != -1) and (len(mapping) > asic_name_index):
-                        asicifname = mapping[asic_name_index]
-                        asic_if_names.append(asicifname)
+                                asic_if_names[alias] = asicifname
+                                asic_if_ids[alias] = "ASIC" + str(asic_id)
+                                if role == "Ext":
+                                    front_panel_asic_ifnames[alias] = asicifname
+                                    front_panel_asic_id[alias] = "ASIC" + str(asic_id)
                     if (speed_index != -1) and (len(mapping) > speed_index):
                         speed = mapping[speed_index]
                         sysport['speed'] = speed
@@ -201,13 +209,16 @@ def get_portmap(self, asic_id=None, include_internal=False,
                     if (num_voq_index != -1) and (len(mapping) > num_voq_index):
                         voq = mapping[num_voq_index]
                         sysport['num_voq'] = voq
-                        sysport['name'] = name
-                        sysport['hostname'] = hostname
-                        sysport['asic_name'] = asic_name
-                        sysport['switchid'] = switchid
                         sysports.append(sysport)
                     if port_index != -1 and len(mapping) > port_index:
                         indexmap[mapping[port_index]] = name
+
+        # Special handling for the Cpu port
+        if include_internal and card_type == "linecard":
+            aliases.append(("Cpu0/{}".format(asic_id if asic_id is not None else 0), -1))
+        if asic_id is not None and card_type == "linecard":
+            asic_if_names["Cpu0/{}".format(asic_id)] = "Cpu0"
+            asic_if_ids["Cpu0/{}".format(asic_id)] = "ASIC" + str(asic_id)
         if len(sysports) > 0:
             sysport = {}
             sysport['name'] = 'Cpu0'
@@ -220,8 +231,9 @@ def get_portmap(self, asic_id=None, include_internal=False,
             sysport['hostname'] = hostname
             sysports.insert(0, sysport)
 
-        return (aliases, portmap, aliasmap, portspeed, front_panel_asic_ifnames, front_panel_asic_id, asic_if_names,
-                sysports, indexmap)
+        return (aliases, front_panel_aliases, inband_aliases, portmap, aliasmap, portspeed,
+                front_panel_asic_ifnames, front_panel_asic_id,
+                asic_if_names, asic_if_ids, sysports, indexmap)
 
 
 def main():
@@ -233,33 +245,46 @@ def main():
             card_type=dict(type='str', required=False),
             hostname=dict(type='str', required=False),
             switchids=dict(type='list', required=False),
-            slotid=dict(type='str', required=False)
+            slotid=dict(type='str', required=False),
+            sort_by_index=dict(type='bool', required=False, default=True)
         ),
         supports_check_mode=True
     )
     m_args = module.params
     try:
-        aliases = []
-        portmap = {}
-        aliasmap = {}
-        portspeed = {}
-        sysports = []
-        indexmap = {}
-        # Map of ASIC interface names to front panel interfaces
-        front_panel_asic_ifnames = {}
-        front_panel_asic_ifs_asic_id = {}
-        # { asic_name: [ asic interfaces] }
-        asic_if_names = {}
+        aliases = []  # list of port aliases
+        front_panel_aliases = []  # list of (front panel port aliases, port indexes)
+        portmap = {}  # name to alias map
+        aliasmap = {}  # alias to name map
+        portspeed = {}  # alias to speed map
+        sysports = []  # list of system ports
+        indexmap = {}  # index to port name map
+        front_panel_asic_ifnames = {}  # Map of interface aliases to interface names for front panel ports
+        front_panel_asic_ifs_asic_id = {}  # Map of interface aliases to asic ids for front panel ports
+        asic_if_names = {}   # Map of interface aliases to interface names for front panel ports
+        asic_if_asic_ids = {}  # Map of interface aliases to asic ids
+        # Chassis related info
+        midplane_port_alias = []  # list of (midplane port aliases, port indexes)
+        inband_port_alias = []  # list of (inband port aliases, port indexes)
 
         if 'card_type' in m_args and m_args['card_type'] == 'supervisor':
+            midplane_port_alias.append(("Midplane", 0))
+            if 'include_internal' in m_args and m_args['include_internal'] is True:
+                aliases.append(("Midplane", -1))
+
             module.exit_json(ansible_facts={'port_alias': aliases,
+                                            'front_panel_port_alias': front_panel_aliases,
+                                            'midplane_port_alias': midplane_port_alias,
+                                            'inband_port_alias': inband_port_alias,
                                             'port_name_map': portmap,
                                             'port_alias_map': aliasmap,
                                             'port_speed': portspeed,
                                             'front_panel_asic_ifnames': [],
                                             'front_panel_asic_ids': [],
-                                            'asic_if_names': asic_if_names,
-                                            'sysports': sysports})
+                                            'asic_if_names': [],
+                                            'asic_if_asic_ids': [],
+                                            'sysports': sysports,
+                                            'port_index_map': indexmap})
             return
         allmap = SonicPortAliasMap(m_args['hwsku'])
         switchids = None
@@ -294,16 +319,32 @@ def main():
         hostname = ""
         if 'hostname' in m_args:
             hostname = m_args['hostname']
+        card_type = None
+        if 'card_type' in m_args:
+            card_type = m_args['card_type']
+
+        if card_type == 'linecard':
+            midplane_port_alias.append(("Midplane", 0))  # midplane port is always the first port (after the mgmt port)
+            if include_internal:
+                aliases.append(("Midplane", -1))
+
+        front_panel_aliases_set = set()
+        inband_port_alias_set = set()
         for asic_id in range(num_asic):
             if switchids and asic_id is not None:
                 switchid = switchids[asic_id]
             if num_asic == 1:
                 asic_id = None
-            (aliases_asic, portmap_asic, aliasmap_asic, portspeed_asic, front_panel_asic, front_panel_asic_ids,
-             asicifnames_asic, sysport_asic, index_name) = allmap.get_portmap(
-                asic_id, include_internal, hostname, switchid, slotid)
+            (aliases_asic, front_panel_aliases_asic, inband_port_alias_asic, portmap_asic, aliasmap_asic,
+             portspeed_asic, front_panel_asic, front_panel_asic_ids,
+             asicifnames_asic, asicifids_asic, sysport_asic, indexmap_asic) = allmap.get_portmap(
+                asic_id, include_internal, hostname, switchid, slotid, card_type)
             if aliases_asic is not None:
                 aliases.extend(aliases_asic)
+            if front_panel_aliases_asic is not None:
+                front_panel_aliases_set.update(front_panel_aliases_asic)
+            if inband_port_alias_asic is not None:
+                inband_port_alias_set.update(inband_port_alias_asic)
             if portmap_asic is not None:
                 portmap.update(portmap_asic)
             if aliasmap_asic is not None:
@@ -315,32 +356,52 @@ def main():
             if front_panel_asic_ids is not None:
                 front_panel_asic_ifs_asic_id.update(front_panel_asic_ids)
             if asicifnames_asic is not None:
-                asic = 'ASIC' + str(asic_id)
-                asic_if_names[asic] = asicifnames_asic
+                asic_if_names.update(asicifnames_asic)
+            if asicifids_asic is not None:
+                asic_if_asic_ids.update(asicifids_asic)
             if sysport_asic is not None:
                 sysports.extend(sysport_asic)
-            if index_name is not None:
-                indexmap.update(index_name)
+            if indexmap_asic is not None:
+                indexmap.update(indexmap_asic)
 
         # Sort the Interface Name needed in multi-asic
-        aliases.sort(key=lambda x: int(x[1]))
+        if m_args['sort_by_index']:
+            # Use the optional argument to enable opt out of sorting by index
+            aliases.sort(key=lambda x: int(x[1]))
+
         # Get ASIC interface names list based on sorted aliases
         front_panel_asic_ifnames_list = []
         front_panel_asic_ifs_asic_id_list = []
+        asic_ifnames_list = []
+        asic_ifs_asic_id_list = []
         for k in aliases:
             if k[0] in front_panel_asic_ifnames:
                 front_panel_asic_ifnames_list.append(
                     front_panel_asic_ifnames[k[0]])
                 front_panel_asic_ifs_asic_id_list.append(
                     front_panel_asic_ifs_asic_id[k[0]])
+            if k[0] in asic_if_names:
+                asic_ifnames_list.append(asic_if_names[k[0]])
+                asic_ifs_asic_id_list.append(asic_if_asic_ids[k[0]])
+
+        # Get front panel and inband interface alias list based on sorted aliases
+        for i, k in enumerate(aliases):
+            if k[0] in front_panel_aliases_set:
+                front_panel_aliases.append((k[0], i))
+            if k[0] in inband_port_alias_set:
+                inband_port_alias.append((k[0], i))
 
         module.exit_json(ansible_facts={'port_alias': [k[0] for k in aliases],
+                                        'front_panel_port_alias': front_panel_aliases,
+                                        'midplane_port_alias': midplane_port_alias,
+                                        'inband_port_alias': inband_port_alias,
                                         'port_name_map': portmap,
                                         'port_alias_map': aliasmap,
                                         'port_speed': portspeed,
                                         'front_panel_asic_ifnames': front_panel_asic_ifnames_list,
                                         'front_panel_asic_ifs_asic_id': front_panel_asic_ifs_asic_id_list,
-                                        'asic_if_names': asic_if_names,
+                                        'asic_if_names': asic_ifnames_list,
+                                        'asic_if_asic_ids': asic_ifs_asic_id_list,
                                         'sysports': sysports,
                                         'port_index_map': indexmap})
 
@@ -349,7 +410,7 @@ def main():
         module.fail_json(msg=fail_msg)
     except Exception as e:
         fail_msg = "failed to find the correct port config for " + \
-            m_args['hwsku'] + str(e)
+            m_args['hwsku'] + "\n" + str(e)
         module.fail_json(msg=fail_msg)
 
 

ansible/roles/vm_set/library/kvm_port.py

@@ -22,9 +22,15 @@ def main():
 
     module = AnsibleModule(argument_spec=dict(
         vmname=dict(required=True),
+        front_panel_port_aliases=dict(required=True, type=list),
+        midplane_port_aliases=dict(required=False, type=list, default=[]),
+        inband_port_aliases=dict(required=False, type=list, default=[]),
     ))
 
     vmname = module.params['vmname']
+    fp_port_aliases = module.params['front_panel_port_aliases']
+    midplane_port_aliases = module.params['midplane_port_aliases']
+    inband_port_aliases = module.params['inband_port_aliases']
 
     try:
         output = subprocess.check_output(
@@ -36,24 +42,37 @@ def main():
 
     mgmt_port = None
     fp_ports = {}
-    cur_fp_idx = 0
+    midplane_ports = []
+    inband_ports = []
 
-    for msg in output.split('\n'):
-        fds = re.split(r'\s+', msg.lstrip())
+    lines = output.split('\n')[2:]  # the first two lines are table headers
+    eth_interfaces = []
+    for line in lines:
+        fds = re.split(r'\s+', line.lstrip())
         if len(fds) != 5:
             continue
         if fds[1] == "ethernet":
-            if mgmt_port is None:
-                mgmt_port = fds[0]
-            else:
-                fp_ports[cur_fp_idx] = fds[0]
-                cur_fp_idx = cur_fp_idx + 1
+            eth_interfaces.append(fds[0])
+
+    if len(eth_interfaces) < 1 + len(fp_port_aliases) + len(midplane_port_aliases) + len(inband_port_aliases):
+        module.fail_json(msg="No enough ethernet ports for {}\n{}\n{}\n{}".format(
+            vmname, fp_port_aliases, midplane_port_aliases, inband_port_aliases))
 
-    if mgmt_port is None:
-        module.fail_json(msg="failed to find mgmt port")
+    # extract mgmt port, fp_ports, midplane_ports(optional), inband_ports(optional)
+    mgmt_port = eth_interfaces[0]
+    eth_interfaces = eth_interfaces[1:]
+    cur_fp_idx = 0
+    for portinfo in fp_port_aliases:
+        fp_ports[cur_fp_idx] = eth_interfaces[portinfo[1]]
+        cur_fp_idx += 1
+    for portinfo in midplane_port_aliases:
+        midplane_ports.append(eth_interfaces[portinfo[1]])
+    for portinfo in inband_port_aliases:
+        inband_ports.append(eth_interfaces[portinfo[1]])
 
     module.exit_json(changed=False, ansible_facts={
-                     'dut_mgmt_port': mgmt_port, 'dut_fp_ports': fp_ports})
+                     'dut_mgmt_port': mgmt_port, 'dut_fp_ports': fp_ports,
+                     'dut_midplane_ports': midplane_ports, 'dut_inband_ports': inband_ports})
 
 
 if __name__ == "__main__":