[minigraph]: Enhancement to minigraph parsing for chassis/multi-asic use case (#14243)

Following changes are done:    

Added Support where if asic configuration is not present in minigraph sonic-cfggen do not error out but instead process it gracefully. 

Use Case: In Supervisor we have number of asic are define as max possible but in minigraph configuration of only valid/available asics only are present. Without this change load_minigraph fails.

Microsoft ADO: 17956325

Signed-off-by: Abhishek Dosi <abdosi@microsoft.com>

Author: abdosi

src/sonic-config-engine/minigraph.py

@@ -456,6 +456,21 @@ def parse_dpg(dpg, hname):
     aclintfs = None
     mgmtintfs = None
     subintfs = None
+    intfs= {}
+    lo_intfs= {}
+    mvrf= {}
+    mgmt_intf= {}
+    voq_inband_intfs= {}
+    vlans= {}
+    vlan_members= {}
+    dhcp_relay_table= {}
+    pcs= {}
+    pc_members= {}
+    acls= {}
+    acl_table_types = {}
+    vni= {}
+    dpg_ecmp_content= {}
+    static_routes= {}
     tunnelintfs = defaultdict(dict)
     tunnelintfs_qos_remap_config = defaultdict(dict)
 
@@ -575,7 +590,6 @@ def parse_dpg(dpg, hname):
                     nexthop = ipnh.find(str(QName(ns, "Address"))).text
                     advertise = ipnh.find(str(QName(ns, "Advertise"))).text
                     static_routes[prefix] = {'nexthop': nexthop, 'ifname': ifname, 'advertise': advertise}
-
             if port_nhipv4_map and port_nhipv6_map:
                 subnet_check_ip = list(port_nhipv4_map.values())[0]
                 for subnet_range in ip_intfs_map:
@@ -659,8 +673,6 @@ def parse_dpg(dpg, hname):
             vlans[sonic_vlan_name] = vlan_attributes
             vlan_member_list[sonic_vlan_name] = vmbr_list
 
-        acls = {}
-        acl_table_types = {}
         for aclintf in aclintfs.findall(str(QName(ns, "AclInterface"))):
             if aclintf.find(str(QName(ns, "InAcl"))) is not None:
                 aclname = aclintf.find(str(QName(ns, "InAcl"))).text.upper().replace(" ", "_").replace("-", "_")
@@ -844,9 +856,8 @@ def parse_dpg(dpg, hname):
                 for table_key, mg_key in tunnel_qos_remap_table_key_to_mg_key_map.items():
                     if mg_key in mg_tunnel.attrib:
                         tunnelintfs_qos_remap_config[tunnel_type][tunnel_name][table_key] = mg_tunnel.attrib[mg_key]
-
         return intfs, lo_intfs, mvrf, mgmt_intf, voq_inband_intfs, vlans, vlan_members, dhcp_relay_table, pcs, pc_members, acls, acl_table_types, vni, tunnelintfs, dpg_ecmp_content, static_routes, tunnelintfs_qos_remap_config
-    return None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None
+    return intfs, lo_intfs, mvrf, mgmt_intf, voq_inband_intfs, vlans, vlan_members, dhcp_relay_table, pcs, pc_members, acls, acl_table_types, vni, tunnelintfs, dpg_ecmp_content, static_routes, tunnelintfs_qos_remap_config
 
 
 def parse_host_loopback(dpg, hname):
@@ -1519,7 +1530,11 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
     if asic_name is None:
         current_device = [devices[key] for key in devices if key.lower() == hostname.lower()][0]
     else:
-        current_device = [devices[key] for key in devices if key.lower() == asic_name.lower()][0]
+        try:
+            current_device = [devices[key] for key in devices if key.lower() == asic_name.lower()][0]
+        except:
+            print("Warning: no asic configuration found for {} in minigraph".format(asic_name), file=sys.stderr)
+            current_device = {}
 
     results = {}
     results['DEVICE_METADATA'] = {'localhost': {
@@ -1845,15 +1860,15 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
     results['PORTCHANNEL_INTERFACE'] = pc_intfs
 
     # for storage backend subinterface info present in minigraph takes precedence over ResourceType
-    if current_device['type'] in backend_device_types and bool(vlan_sub_intfs):
+    if current_device and current_device['type'] in backend_device_types and bool(vlan_sub_intfs):
         del results['INTERFACE']
         del results['PORTCHANNEL_INTERFACE']
         is_storage_device = True
         results['VLAN_SUB_INTERFACE'] = vlan_sub_intfs
         # storage backend T0 have all vlan members tagged
         for vlan in vlan_members:
             vlan_members[vlan]["tagging_mode"] = "tagged"
-    elif current_device['type'] in backend_device_types and (resource_type is None or 'Storage' in resource_type):
+    elif current_device and current_device['type'] in backend_device_types and (resource_type is None or 'Storage' in resource_type):
         del results['INTERFACE']
         del results['PORTCHANNEL_INTERFACE']
         is_storage_device = True
@@ -1930,7 +1945,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
     results['TACPLUS_SERVER'] = dict((item, {'priority': '1', 'tcp_port': '49'}) for item in tacacs_servers)
     if len(acl_table_types) > 0:
         results['ACL_TABLE_TYPE'] = acl_table_types
-    results['ACL_TABLE'] = filter_acl_table_bindings(acls, neighbors, pcs, pc_members, sub_role, current_device['type'], is_storage_device, vlan_members)
+    results['ACL_TABLE'] = filter_acl_table_bindings(acls, neighbors, pcs, pc_members, sub_role, current_device['type'] if current_device else None, is_storage_device, vlan_members)
     results['FEATURE'] = {
         'telemetry': {
             'state': 'enabled'
@@ -1983,18 +1998,18 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
     #     results['MIRROR_SESSION'] = mirror_sessions
 
     # Special parsing for spine chassis frontend routers
-    if current_device['type'] == spine_chassis_frontend_role:
+    if current_device and current_device['type'] == spine_chassis_frontend_role:
         parse_spine_chassis_fe(results, vni, lo_intfs, phyport_intfs, pc_intfs, pc_members, devices)
 
     # Enable console management feature for console swtich
     results['CONSOLE_SWITCH'] = {
         'console_mgmt' : {
-            'enabled' : 'yes' if current_device['type'] in console_device_types else 'no'
+            'enabled' : 'yes' if current_device and current_device['type'] in console_device_types else 'no'
         }
     }
 
     # Enable DHCP Server feature for specific device type
-    if current_device['type'] in dhcp_server_enabled_device_types:
+    if current_device and current_device['type'] in dhcp_server_enabled_device_types:
         results['DEVICE_METADATA']['localhost']['dhcp_server'] = 'enabled'
 
     return results

src/sonic-config-engine/tests/mock_tables/asic4/database_config.json

@@ -0,0 +1,52 @@
+{
+    "INSTANCES": {
+        "redis": {
+            "hostname" : "127.0.0.1",
+            "port" : 6379,
+            "unix_socket_path" : "/var/run/redis/redis.sock"
+        }
+    },
+    "DATABASES" : {
+        "APPL_DB" : {
+            "id" : 0,
+            "separator": ":",
+            "instance" : "redis"
+        },
+        "ASIC_DB" : {
+            "id" : 1,
+            "separator": ":",
+            "instance" : "redis"
+        },
+        "COUNTERS_DB" : {
+            "id" : 2,
+            "separator": ":",
+            "instance" : "redis"
+        },
+        "CONFIG_DB" : {
+            "id" : 4,
+            "separator": "|",
+            "instance" : "redis"
+        },
+        "PFC_WD_DB" : {
+            "id" : 5,
+            "separator": ":",
+            "instance" : "redis"
+        },
+        "FLEX_COUNTER_DB" : {
+            "id" : 5,
+            "separator": ":",
+            "instance" : "redis"
+        },
+        "STATE_DB" : {
+            "id" : 6,
+            "separator": "|",
+            "instance" : "redis"
+        },
+        "SNMP_OVERLAY_DB" : {
+            "id" : 7,
+            "separator": "|",
+            "instance" : "redis"
+        }
+    },
+    "VERSION" : "1.1"
+}

src/sonic-config-engine/tests/mock_tables/database_global.json

@@ -18,6 +18,10 @@
         {
             "namespace" : "asic3",
             "include" : "./asic3/database_config.json"
+        },
+        {
+            "namespace" : "asic4",
+            "include" : "./asic4/database_config.json"
         }
     ],
     "VERSION" : "1.0"

src/sonic-config-engine/tests/multi_npu_data/sample_port_config-4.ini

@@ -0,0 +1,2 @@
+# name              lanes               alias          index      asic_port_name    role
+Ethernet-BP416     29,30,31,32      Eth0-ASIC4         16         Eth0-ASIC4        Int

src/sonic-config-engine/tests/test_multinpu_cfggen.py

@@ -30,12 +30,14 @@ def setUp(self):
         self.port_config = []
         for asic in range(NUM_ASIC):
             self.port_config.append(os.path.join(self.test_data_dir, "sample_port_config-{}.ini".format(asic)))
+        self.sample_no_asic_port_config = os.path.join(self.test_data_dir, 'sample_port_config-4.ini')
         self.output_file = os.path.join(self.test_dir, 'output')
         os.environ["CFGGEN_UNIT_TESTING"] = "2"
 
-    def run_script(self, argument, check_stderr=False, output_file=None):
+    def run_script(self, argument, check_stderr=True, output_file=None, validateYang=True):
         print('\n    Running sonic-cfggen ' + ' '.join(argument))
-        self.assertTrue(self.yang.validate(argument))
+        if validateYang:
+            self.assertTrue(self.yang.validate(argument))
         if check_stderr:
             output = subprocess.check_output(self.script_file + argument, stderr=subprocess.STDOUT)
         else:
@@ -542,5 +544,10 @@ def test_bgpd_frr_frontendasic(self):
     def test_bgpd_frr_backendasic(self):
         self.assertTrue(*self.run_frr_asic_case('bgpd/bgpd.conf.j2', 'bgpd_frr_backend_asic.conf', "asic3", self.port_config[3]))
 
+    def test_no_asic_in_graph(self):
+        argument = ["-m", self.sample_graph, "-p", self.sample_no_asic_port_config, "-n", "asic4", "--var-json", "PORTCHANNEL"]
+        output = json.loads(self.run_script(argument, check_stderr=False, validateYang=False))
+        self.assertDictEqual(output, {})
+
     def tearDown(self):
         os.environ["CFGGEN_UNIT_TESTING"] = ""