[tlm_teamd]: Try to add LAG again, when teamd is not ready first time (sonic-net#1347)

* Postpone add LAG operation, when can't add right away

* Set empty log function for teamdctl library

* Return empty list if there is no ports in the PortChannel

Co-authored-by: PS <ps@cs.com>

Author: pavel-shirshov

tlm_teamd/main.cpp

@@ -107,6 +107,7 @@ int main()
             }
             else if (res == swss::Select::TIMEOUT)
             {
+                teamdctl_mgr.process_add_queue();
                 values_store.update(teamdctl_mgr.get_dumps());
             }
             else

tlm_teamd/teamdctl_mgr.cpp

@@ -1,9 +1,28 @@
 #include <cstring>
+#include <algorithm>
 
 #include <logger.h>
 
 #include "teamdctl_mgr.h"
 
+///
+/// Custom function for libteamdctl logger. IT is empty to prevent libteamdctl to spam us with the error messages
+/// @param tdc teamdctl descriptor
+/// @param priority priority of the message
+/// @param file file where error was raised
+/// @param line line in the file where error was raised
+/// @param fn function where the error was raised
+/// @param format format of the error message
+/// @param args arguments of the error message
+void teamdctl_log_function(struct teamdctl *tdc, int priority,
+                           const char *file, int line,
+                           const char *fn, const char *format,
+                           va_list args)
+{
+
+}
+
+
 ///
 /// The destructor clean up handlers to teamds
 ///
@@ -31,10 +50,11 @@ bool TeamdCtlMgr::has_key(const std::string & lag_name) const
 }
 
 ///
-/// Adds a LAG interface with lag_name to the manager
-/// This method allocates structures to connect to teamd
+/// Public method to add a LAG interface with lag_name to the manager
+/// This method tries to add. If the method can't add the LAG interface,
+/// this action will be postponed.
 /// @param lag_name a name for LAG interface
-/// @return true if the lag was added successfully, false otherwise
+/// @return true if the lag was added or postponed successfully, false otherwise
 ///
 bool TeamdCtlMgr::add_lag(const std::string & lag_name)
 {
@@ -43,26 +63,51 @@ bool TeamdCtlMgr::add_lag(const std::string & lag_name)
         SWSS_LOG_DEBUG("The LAG '%s' was already added. Skip adding it.", lag_name.c_str());
         return true;
     }
-    else
+    return try_add_lag(lag_name);
+}
+
+///
+/// Try to adds a LAG interface with lag_name to the manager
+/// This method allocates structures to connect to teamd
+/// if the method can't add, it will retry to add next time
+/// @param lag_name a name for LAG interface
+/// @return true if the lag was added successfully, false otherwise
+///
+bool TeamdCtlMgr::try_add_lag(const std::string & lag_name)
+{
+    if (m_lags_to_add.find(lag_name) == m_lags_to_add.end())
     {
-        auto tdc = teamdctl_alloc();
-        if (!tdc)
-        {
-            SWSS_LOG_ERROR("Can't allocate memory for teamdctl handler. LAG='%s'", lag_name.c_str());
-            return false;
-        }
+        m_lags_to_add[lag_name] = 0;
+    }
 
-        int err = teamdctl_connect(tdc, lag_name.c_str(), nullptr, nullptr);
-        if (err)
+    int attempt = m_lags_to_add[lag_name];
+
+    auto tdc = teamdctl_alloc();
+    if (!tdc)
+    {
+        SWSS_LOG_ERROR("Can't allocate memory for teamdctl handler. LAG='%s'. attempt=%d", lag_name.c_str(), attempt);
+        m_lags_to_add[lag_name]++;
+        return false;
+    }
+
+    teamdctl_set_log_fn(tdc, &teamdctl_log_function);
+
+    int err = teamdctl_connect(tdc, lag_name.c_str(), nullptr, nullptr);
+    if (err)
+    {
+        if (attempt != 0)
         {
-            SWSS_LOG_ERROR("Can't connect to teamd LAG='%s', error='%s'", lag_name.c_str(), strerror(-err));
-            teamdctl_free(tdc);
-            return false;
+            SWSS_LOG_WARN("Can't connect to teamd LAG='%s', error='%s'. attempt=%d", lag_name.c_str(), strerror(-err), attempt);
         }
-        m_handlers.emplace(lag_name, tdc);
-        SWSS_LOG_NOTICE("The LAG '%s' has been added.", lag_name.c_str());
+        teamdctl_free(tdc);
+        m_lags_to_add[lag_name]++;
+        return false;
     }
 
+    m_handlers.emplace(lag_name, tdc);
+    m_lags_to_add.erase(lag_name);
+    SWSS_LOG_NOTICE("The LAG '%s' has been added.", lag_name.c_str());
+
     return true;
 }
 
@@ -82,13 +127,39 @@ bool TeamdCtlMgr::remove_lag(const std::string & lag_name)
         m_handlers.erase(lag_name);
         SWSS_LOG_NOTICE("The LAG '%s' has been removed.", lag_name.c_str());
     }
+    else if (m_lags_to_add.find(lag_name) != m_lags_to_add.end())
+    {
+        m_lags_to_add.erase(lag_name);
+        SWSS_LOG_DEBUG("The LAG '%s' has been removed from adding queue.", lag_name.c_str());
+    }
     else
     {
         SWSS_LOG_WARN("The LAG '%s' hasn't been added. Can't remove it", lag_name.c_str());
     }
     return true;
 }
 
+///
+/// Process the queue with postponed add operations for LAG.
+///
+void TeamdCtlMgr::process_add_queue()
+{
+    std::vector<std::string> lag_names_to_add;
+    std::transform(m_lags_to_add.begin(), m_lags_to_add.end(), lag_names_to_add.begin(), [](auto pair) { return pair.first; });
+    for (const auto lag_name: lag_names_to_add)
+    {
+        bool result = try_add_lag(lag_name);
+        if (!result)
+        {
+            if (m_lags_to_add[lag_name] == TeamdCtlMgr::max_attempts_to_add)
+            {
+                SWSS_LOG_ERROR("Can't connect to teamd after %d attempts. LAG '%s'", TeamdCtlMgr::max_attempts_to_add, lag_name.c_str());
+                m_lags_to_add.erase(lag_name);
+            }
+        }
+    }
+}
+
 ///
 /// Get json dump from teamd for LAG interface with name lag_name
 /// @param lag_name a name for LAG interface

tlm_teamd/teamdctl_mgr.h

@@ -15,13 +15,18 @@ class TeamdCtlMgr
 public:
     TeamdCtlMgr() = default;
     ~TeamdCtlMgr();
-    bool add_lag(const std::string & kag_name);
-    bool remove_lag(const std::string & kag_name);
+    bool add_lag(const std::string & lag_name);
+    bool remove_lag(const std::string & lag_name);
+    void process_add_queue();
     TeamdCtlDump get_dump(const std::string & lag_name);
     TeamdCtlDumps get_dumps();
 
 private:
     bool has_key(const std::string & lag_name) const;
+    bool try_add_lag(const std::string & lag_name);
 
     std::unordered_map<std::string, struct teamdctl*> m_handlers;
+    std::unordered_map<std::string, int> m_lags_to_add;
+
+    const int max_attempts_to_add = 10;
 };

tlm_teamd/values_store.cpp

@@ -16,7 +16,7 @@ std::vector<std::string> ValuesStore::get_ports(json_t * root)
     int err = json_unpack(root, "{s:o}", "ports", &ports);
     if (err != 0)
     {
-        throw std::runtime_error("Can't find 'ports' in the json dump object");
+        return result;
     }
 
     const char * key;