Remove rsyslog plugin from all non eventd containers (sonic-net#21338)

#### Why I did it

Fixes Issue sonic-net#21227

- As of current, we have rsyslog_plugin and corresponding regex and configuration files in containers like bgp, swss, dhcp_relay. These binaries and files are built as part of the docker container itself. 

- This change removes that dependency by storing all rsyslog_plugin regex and conference files in the eventd docker container, and at runtime copying them to the host. 

- Fixes the issue that some images will not be able to publish certain events like "bgp-state", or "dhcp-relay-discard" due to reuse of docker containers during where eventd was disabled.

#### How I did it

- Removed all rsyslog plugin configuration and regex files from containers such as bgp, swss, dhcp_relay. All files for configuration and regexes are built inside eventd docker container. When eventd docker is started, it will copy these files to the host and restart rsyslog.
- sshd regex and zebra regex are removed because those events are currently not being supported.
- Possible race condition with rsyslog-config is avoided due to "After" and since rsyslog-config.service is type oneshot, rsyslog-config.sh is guaranteed to complete before eventd.service is called.

#### How to verify it

Tested on all elastic pipeline for 6 different HWSKUs

Arista-7260CX, Arista720DT, Mellanox2700, Cisco 8102, Nokia M0, Arista 7060CX

Author: zbud-msft

dockers/docker-config-engine-bookworm/00-load-omprog.conf

@@ -47,7 +47,6 @@ RUN pip3 install redis==5.0.1
 COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
 COPY ["files/readiness_probe.sh", "/usr/bin/"]
 COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
-COPY ["00-load-omprog.conf", "/etc/rsyslog.d/"]
 
 ## Clean up
 RUN apt-get purge -y               \

dockers/docker-config-engine-bookworm/Dockerfile.j2

@@ -50,7 +50,6 @@ RUN pip3 install redis==4.5.4
 COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
 COPY ["files/readiness_probe.sh", "/usr/bin/"]
 COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
-COPY ["00-load-omprog.conf", "/etc/rsyslog.d/"]
 
 ## Clean up
 RUN apt-get purge -y               \

dockers/docker-config-engine-bullseye/00-load-omprog.conf

@@ -50,14 +50,4 @@ COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor"]
 COPY ["cli", "/cli/"]
 
-{% if include_system_eventd == "y" and build_reduce_image_size != "y" %}
-# Copy regex json and rsyslog_plugin.conf file into rsyslog.d
-COPY ["*.json", "/etc/rsyslog.d/"]
-COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
-# Create dhcp_relay_regex.conf
-RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/dhcp_relay_events.conf
-RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2
-RUN rm -f /etc/rsyslog.d/events_info.json
-{% endif %}
-
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-config-engine-bullseye/Dockerfile.j2

@@ -28,9 +28,27 @@ RUN apt-get clean -y         && \
     apt-get autoremove -y    && \
     rm -rf /debs
 
+RUN mkdir -p /etc/rsyslog.d/rsyslog_plugin_conf
+
 COPY ["start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor"]
+COPY ["*.json", "/etc/rsyslog.d/rsyslog_plugin_conf/"]
+COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/rsyslog_plugin_conf/"]
+
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin_conf/rsyslog_plugin.conf.j2 /etc/rsyslog.d/rsyslog_plugin_conf/host_events_info.json > /etc/rsyslog.d/rsyslog_plugin_conf/host_events.conf
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin_conf/rsyslog_plugin.conf.j2 /etc/rsyslog.d/rsyslog_plugin_conf/bgp_events_info.json > /etc/rsyslog.d/rsyslog_plugin_conf/bgp_events.conf
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin_conf/rsyslog_plugin.conf.j2 /etc/rsyslog.d/rsyslog_plugin_conf/dhcp_relay_events_info.json > /etc/rsyslog.d/rsyslog_plugin_conf/dhcp_relay_events.conf
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin_conf/rsyslog_plugin.conf.j2 /etc/rsyslog.d/rsyslog_plugin_conf/swss_events_info.json > /etc/rsyslog.d/rsyslog_plugin_conf/swss_events.conf
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin_conf/rsyslog_plugin.conf.j2 /etc/rsyslog.d/rsyslog_plugin_conf/syncd_events_info.json > /etc/rsyslog.d/rsyslog_plugin_conf/syncd_events.conf
+
+
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin_conf/rsyslog_plugin.conf.j2
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin_conf/host_events_info.json
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin_conf/bgp_events_info.json
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin_conf/dhcp_relay_events_info.json
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin_conf/swss_events_info.json
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin_conf/syncd_events_info.json
 
 ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-dhcp-relay/Dockerfile.j2

@@ -5,10 +5,6 @@
             "name": "monit",
             "parse_json": "monit_regex.json"
         },
-        {
-            "name": "sshd",
-            "parse_json": "sshd_regex.json"
-        },
         {
             "name": "systemd",
             "parse_json": "systemd_regex.json"

dockers/docker-eventd/Dockerfile.j2

@@ -3,4 +3,3 @@
 if [ "${RUNTIME_OWNER}" == "" ]; then
     RUNTIME_OWNER="kube"
 fi
-

dockers/docker-fpm-frr/events_info.json dockers/docker-eventd/bgp_events_info.json

@@ -59,12 +59,4 @@ RUN chmod a+x /usr/bin/TSA && \
     chmod a+x /usr/bin/TSC && \
     chmod a+x /usr/bin/zsocket.sh
 
-{% if include_system_eventd == "y" and build_reduce_image_size != "y" %}
-COPY ["*.json", "/etc/rsyslog.d/"]
-COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
-RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/bgp_events.conf
-RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2
-RUN rm -f /etc/rsyslog.d/events_info.json
-{% endif %}
-
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-fpm-frr/bgp_regex.json dockers/docker-eventd/bgp_regex.json

@@ -80,14 +80,4 @@ RUN sonic-cfggen -a "{\"ENABLE_ASAN\":\"{{ENABLE_ASAN}}\"}" -t /usr/share/sonic/
 RUN rm -f /usr/share/sonic/templates/docker-init.j2
 RUN chmod 755 /usr/bin/docker-init.sh
 
-{% if include_system_eventd == "y" and build_reduce_image_size != "y" %}
-# Copy all regex json files and rsyslog_plugin.conf to rsyslog.d
-COPY ["*.json", "/etc/rsyslog.d/"]
-COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
-# Create swss rsyslog_plugin conf file
-RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/swss_events.conf
-RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2
-RUN rm -f /etc/rsyslog.d/events_info.json
-{% endif %}
-
 ENTRYPOINT ["/usr/bin/docker-init.sh"]

files/build_templates/bgpd_regex.json dockers/docker-eventd/bgpd_regex.json

@@ -339,6 +339,13 @@ function postStartAction()
             docker cp $PSENSOR pmon:/usr/bin/
         fi
     fi
+{%- elif docker_container_name == "eventd" %}
+    HOST_RSYSLOG_PLUGIN_CONF_FILE="/etc/rsyslog.d/host_events.conf"
+    if [ ! -f ${HOST_RSYSLOG_PLUGIN_CONF_FILE} ]; then
+        for f in $(docker exec -i eventd ls /etc/rsyslog.d/rsyslog_plugin_conf); do docker cp eventd:/etc/rsyslog.d/rsyslog_plugin_conf/$f /etc/rsyslog.d/; done
+        systemctl reset-failed rsyslog
+        systemctl restart rsyslog
+    fi
 {%- else %}
     : # nothing
 {%- endif %}

dockers/docker-dhcp-relay/events_info.json dockers/docker-eventd/dhcp_relay_events_info.json

@@ -1,7 +1,7 @@
 [Unit]
 Description=EVENTD container
 Requires=config-setup.service
-After=config-setup.service
+After=rsyslog-config.service
 BindsTo=sonic.target
 After=sonic.target
 StartLimitIntervalSec=1200

dockers/docker-dhcp-relay/dhcp_relay_regex.json dockers/docker-eventd/dhcp_relay_regex.json

@@ -367,22 +367,6 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-db-cli_*.deb || \
 # Install sonic-rsyslog-plugin
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-rsyslog-plugin_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
-
-# Generate host conf for rsyslog_plugin
-j2 -f json $BUILD_TEMPLATES/rsyslog_plugin.conf.j2 $BUILD_TEMPLATES/events_info.json | sudo tee $FILESYSTEM_ROOT_ETC/rsyslog.d/host_events.conf
-sudo cp $BUILD_TEMPLATES/monit_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/sshd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/systemd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/kernel_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/dockerd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/seu_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/zebra_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-sudo cp $BUILD_TEMPLATES/bgpd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-
-
-j2 -f json $BUILD_TEMPLATES/rsyslog_plugin.conf.j2 $BUILD_TEMPLATES/syncd_events_info.json | sudo tee $FILESYSTEM_ROOT_ETC/rsyslog.d/syncd_events.conf
-sudo cp $BUILD_TEMPLATES/syncd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
-
 {% endif %}
 
 # Install custom-built monit package and SONiC configuration files

files/build_templates/dockerd_regex.json dockers/docker-eventd/dockerd_regex.json

@@ -13,12 +13,6 @@ $ModLoad omprog
 
 if re_match($programname, "bgp[0-9]*#(frr|zebra|staticd|watchfrr)") then {
     /var/log/frr/zebra.log
-    {% if include_system_eventd == "y" and build_reduce_image_size != "y" %}
-    action(type="omprog"
-        binary="/usr/bin/rsyslog_plugin -r /etc/rsyslog.d/zebra_regex.json -m sonic-events-bgp"
-        output="/var/log/rsyslog_plugin.log"
-        template="prog_msg")
-    {% endif %}
     stop
 }
 

files/build_templates/events_info.json dockers/docker-eventd/host_events_info.json

@@ -6,10 +6,10 @@ DOCKER_DHCP_RELAY_DBG = $(DOCKER_DHCP_RELAY_STEM)-$(DBG_IMAGE_MARK).gz
 
 $(DOCKER_DHCP_RELAY)_PATH = $(DOCKERS_PATH)/$(DOCKER_DHCP_RELAY_STEM)
 
-$(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_RELAY) $(SONIC_DHCPMON) $(SONIC_DHCPRELAY) $(LIBSWSSCOMMON) $(SONIC_RSYSLOG_PLUGIN) 
+$(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_RELAY) $(SONIC_DHCPMON) $(SONIC_DHCPRELAY) $(LIBSWSSCOMMON)
 
 $(DOCKER_DHCP_RELAY)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BOOKWORM)_DBG_DEPENDS)
-$(DOCKER_DHCP_RELAY)_DBG_DEPENDS += $(ISC_DHCP_RELAY_DBG) $(SONIC_DHCPRELAY_DBG) $(SONIC_DHCPMON_DBG) $(SONIC_RSYSLOG_PLUGIN)
+$(DOCKER_DHCP_RELAY)_DBG_DEPENDS += $(ISC_DHCP_RELAY_DBG) $(SONIC_DHCPRELAY_DBG) $(SONIC_DHCPMON_DBG)
 
 $(DOCKER_DHCP_RELAY)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BOOKWORM)_DBG_IMAGE_PACKAGES)
 

files/build_templates/kernel_regex.json dockers/docker-eventd/kernel_regex.json

@@ -6,7 +6,7 @@ DOCKER_DHCP_SERVER_DBG = $(DOCKER_DHCP_SERVER_STEM)-$(DBG_IMAGE_MARK).gz
 
 $(DOCKER_DHCP_SERVER)_PATH = $(DOCKERS_PATH)/$(DOCKER_DHCP_SERVER_STEM)
 
-$(DOCKER_DHCP_SERVER)_DEPENDS = $(LIBSWSSCOMMON) $(SONIC_RSYSLOG_PLUGIN)
+$(DOCKER_DHCP_SERVER)_DEPENDS = $(LIBSWSSCOMMON)
 $(DOCKER_DHCP_SERVER)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BOOKWORM)_DBG_DEPENDS)
 
 $(DOCKER_DHCP_SERVER)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BOOKWORM)_DBG_IMAGE_PACKAGES)

files/build_templates/monit_regex.json dockers/docker-eventd/monit_regex.json

@@ -35,7 +35,7 @@ $(DOCKER_EVENTD)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
 $(DOCKER_EVENTD)_CONTAINER_NAME = eventd
 $(DOCKER_EVENTD)_RUN_OPT += -t
 $(DOCKER_EVENTD)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
-$(DOCKER_EVENTD)_RUN_OPT += -v /etc/localtime:/etc/localtime:ro 
+$(DOCKER_EVENTD)_RUN_OPT += -v /etc/localtime:/etc/localtime:ro
 
 SONIC_BOOKWORM_DOCKERS += $(DOCKER_EVENTD)
 SONIC_BOOKWORM_DBG_DOCKERS += $(DOCKER_EVENTD_DBG)

files/build_templates/seu_regex.json dockers/docker-eventd/seu_regex.json

@@ -7,10 +7,10 @@ DOCKER_FPM_FRR_DBG = $(DOCKER_FPM_FRR_STEM)-$(DBG_IMAGE_MARK).gz
 $(DOCKER_FPM_FRR)_PATH = $(DOCKERS_PATH)/$(DOCKER_FPM_FRR_STEM)
 $(DOCKER_FPM_FRR)_PYTHON_WHEELS += $(SONIC_BGPCFGD) $(SONIC_FRR_MGMT_FRAMEWORK)
 
-$(DOCKER_FPM_FRR)_DEPENDS += $(FRR) $(FRR_SNMP) $(SWSS) $(LIBYANG2) $(SONIC_RSYSLOG_PLUGIN)
+$(DOCKER_FPM_FRR)_DEPENDS += $(FRR) $(FRR_SNMP) $(SWSS) $(LIBYANG2)
 $(DOCKER_FPM_FRR)_DBG_DEPENDS = $($(DOCKER_SWSS_LAYER_BOOKWORM)_DBG_DEPENDS)
 $(DOCKER_FPM_FRR)_DBG_DEPENDS += $(SWSS_DBG) $(LIBSWSSCOMMON_DBG) \
-                                $(FRR_DBG) $(FRR_SNMP_DBG) $(LIBYANG2_DBG) $(SONIC_RSYSLOG_PLUGIN)
+                                $(FRR_DBG) $(FRR_SNMP_DBG) $(LIBYANG2_DBG)
 
 $(DOCKER_FPM_FRR)_DBG_IMAGE_PACKAGES = $($(DOCKER_SWSS_LAYER_BOOKWORM)_DBG_IMAGE_PACKAGES)
 

dockers/docker-eventd/start.sh

@@ -4,7 +4,7 @@ DOCKER_ORCHAGENT_STEM = docker-orchagent
 DOCKER_ORCHAGENT = $(DOCKER_ORCHAGENT_STEM).gz
 DOCKER_ORCHAGENT_DBG = $(DOCKER_ORCHAGENT_STEM)-$(DBG_IMAGE_MARK).gz
 
-$(DOCKER_ORCHAGENT)_DEPENDS += $(SWSS) $(SONIC_RSYSLOG_PLUGIN) $(LIB_SONIC_DASH_API)
+$(DOCKER_ORCHAGENT)_DEPENDS += $(SWSS) $(LIB_SONIC_DASH_API)
 
 ifeq ($(ENABLE_ASAN), y)
 $(DOCKER_ORCHAGENT)_DEPENDS += $(SWSS_DBG)
@@ -13,8 +13,7 @@ endif
 $(DOCKER_ORCHAGENT)_DBG_DEPENDS = $($(DOCKER_SWSS_LAYER_BOOKWORM)_DBG_DEPENDS)
 $(DOCKER_ORCHAGENT)_DBG_DEPENDS +=   $(SWSS_DBG) \
                                 $(LIBSWSSCOMMON_DBG) \
-                                $(LIBSAIREDIS_DBG) \
-				$(SONIC_RSYSLOG_PLUGIN)
+                                $(LIBSAIREDIS_DBG)
 $(DOCKER_ORCHAGENT)_PYTHON_WHEELS += $(SCAPY)
 
 $(DOCKER_ORCHAGENT)_DBG_IMAGE_PACKAGES = $($(DOCKER_SWSS_LAYER_BOOKWORM)_DBG_IMAGE_PACKAGES)