Skip to content

Commit b7f84ed

Browse files
rkavitha-hclVSuryaprasad-HCL
rkavitha-hcl
authored and
VSuryaprasad-HCL
committed
gNOI Warm Reboot - Added tests
1 parent 6c2576a commit b7f84ed

File tree

3 files changed

+247
-2
lines changed

3 files changed

+247
-2
lines changed

‎src/sonic-framework/rebootbackend/rebootbe.cpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ RebootBE::RebootBE(DbusInterface &dbus_interface)
3030
}
3131

3232
RebootBE::RebManagerStatus RebootBE::GetCurrentStatus() {
33-
const std::lock_guard<std::mutex> lock(m_StatusMutex);
3433
return m_CurrentStatus;
3534
}
3635

@@ -42,6 +41,9 @@ void RebootBE::SetCurrentStatus(RebManagerStatus newStatus) {
4241
void RebootBE::Start() {
4342
SWSS_LOG_ENTER();
4443
SWSS_LOG_NOTICE("--- Starting rebootbackend ---");
44+
swss::WarmStart::initialize("rebootbackend", "sonic-framework");
45+
swss::WarmStart::checkWarmStart("rebootbackend", "sonic-framework",
46+
/*incr_restore_cnt=*/false);
4547

4648
swss::Select s;
4749
s.addSelectable(&m_NotificationConsumer);

‎src/sonic-framework/tests/reboot_thread_test.cpp

+71
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,38 @@ TEST_F(RebootStatusTest, TestGetStatus) {
9191
EXPECT_EQ(0, response.when());
9292
}
9393

94+
TEST_F(RebootStatusTest, TestGetWarmStatus) {
95+
std::chrono::nanoseconds curr_ns =
96+
std::chrono::high_resolution_clock::now().time_since_epoch();
97+
98+
m_status.set_start_status(gnoi::system::RebootMethod::WARM, "reboot because");
99+
100+
gnoi::system::RebootStatusResponse response = m_status.get_response();
101+
EXPECT_EQ(
102+
response.status().status(),
103+
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN);
104+
105+
m_status.set_completed_status(
106+
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS,
107+
"anything");
108+
109+
response = m_status.get_response();
110+
111+
// message should be empty while reboot is active
112+
EXPECT_THAT(response.status().message(), StrEq(""));
113+
114+
uint64_t reboot_ns = response.when();
115+
EXPECT_TRUE(reboot_ns > (uint64_t)curr_ns.count());
116+
117+
m_status.set_inactive();
118+
response = m_status.get_response();
119+
EXPECT_THAT(response.status().message(), StrEq("anything"));
120+
EXPECT_EQ(
121+
response.status().status(),
122+
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS);
123+
EXPECT_EQ(0, response.when());
124+
}
125+
94126
class RebootThreadTest : public ::testing::Test {
95127
protected:
96128
RebootThreadTest()
@@ -285,4 +317,43 @@ TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) {
285317
""));
286318
}
287319

320+
TEST_F(RebootThreadTest, TestNoWarmIfNonRetriableFailure) {
321+
set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot");
322+
set_completed_status(
323+
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
324+
"failed to warm reboot");
325+
force_inactive();
326+
327+
gnoi::system::RebootRequest request;
328+
request.set_method(gnoi::system::RebootMethod::WARM);
329+
330+
NotificationResponse response = m_reboot_thread.Start(request);
331+
EXPECT_EQ(response.status, swss::StatusCode::SWSS_RC_FAILED_PRECONDITION);
332+
EXPECT_EQ(response.json_string,
333+
"RebootThread: last WARM reboot failed with non-retriable failure");
334+
}
335+
336+
TEST_F(RebootThreadTest, TestSigTermStartofDoReboot) {
337+
sigterm_requested = true;
338+
set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot");
339+
do_reboot();
340+
force_inactive();
341+
gnoi::system::RebootStatusResponse response = m_reboot_thread.GetResponse();
342+
EXPECT_THAT(
343+
response,
344+
IsStatus(
345+
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN,
346+
""));
347+
}
348+
349+
TEST_F(RebootThreadTest, TestWaitForRebootPositive) {
350+
overwrite_reboot_timeout(1);
351+
set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot");
352+
swss::Select s;
353+
swss::SelectableEvent m_stop;
354+
s.addSelectable(&m_stop);
355+
RebootThread::Progress progress = wait_for_platform_reboot(s);
356+
EXPECT_EQ(progress, RebootThread::Progress::PROCEED);
357+
}
358+
288359
} // namespace rebootbackend

‎src/sonic-framework/tests/rebootbe_test.cpp

+173-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ namespace rebootbackend {
2323
#define TWO_SECONDS (2)
2424
#define TENTH_SECOND_MS (100)
2525
#define SELECT_TIMEOUT_250_MS (250)
26+
#define ONE_SECOND_MS (1000)
27+
#define TWO_SECONDS_MS (2000)
2628

2729
namespace gpu = ::google::protobuf::util;
2830
using namespace gnoi::system;
@@ -59,6 +61,7 @@ class RebootBETestWithoutStop : public ::testing::Test {
5961
m_rebootbeReponseChannel(&m_db, REBOOT_RESPONSE_NOTIFICATION_CHANNEL),
6062
m_rebootbe(m_dbus_interface) {
6163
sigterm_requested = false;
64+
// TestUtils::clear_tables(m_db);
6265

6366
m_s.addSelectable(&m_rebootbeReponseChannel);
6467

@@ -69,6 +72,15 @@ class RebootBETestWithoutStop : public ::testing::Test {
6972
}
7073
virtual ~RebootBETestWithoutStop() = default;
7174

75+
void force_warm_start_state(bool enabled) {
76+
swss::Table enable_table(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME);
77+
enable_table.hset("system", "enable", enabled ? "true" : "false");
78+
enable_table.hset("sonic-framework", "enable", enabled ? "true" : "false");
79+
80+
swss::Table restart_table(&m_db, STATE_WARM_RESTART_TABLE_NAME);
81+
restart_table.hset("rebootbackend", "restore_count", enabled ? "0" : "");
82+
}
83+
7284
void start_rebootbe() {
7385
m_rebootbe_thread =
7486
std::make_unique<std::thread>(&RebootBE::Start, &m_rebootbe);
@@ -194,6 +206,48 @@ class RebootBETest : public RebootBETestWithoutStop {
194206
}
195207
};
196208

209+
TEST_F(RebootBETest, WarmbootInProgressBlocksNewWarmboot) {
210+
force_warm_start_state(true);
211+
212+
start_rebootbe();
213+
std::this_thread::sleep_for(std::chrono::milliseconds(100));
214+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
215+
RebootBE::RebManagerStatus::WARM_INIT_WAIT);
216+
217+
// Send a warmboot request, confirm it fails.
218+
RebootRequest request;
219+
request.set_method(RebootMethod::WARM);
220+
start_reboot_via_rpc(request, swss::StatusCode::SWSS_RC_IN_USE);
221+
222+
std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
223+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
224+
RebootBE::RebManagerStatus::WARM_INIT_WAIT);
225+
force_warm_start_state(false);
226+
}
227+
228+
TEST_F(RebootBETest, ColdbootWhileWarmbootInProgress) {
229+
force_warm_start_state(true);
230+
set_mock_defaults();
231+
232+
start_rebootbe();
233+
std::this_thread::sleep_for(std::chrono::milliseconds(100));
234+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
235+
RebootBE::RebManagerStatus::WARM_INIT_WAIT);
236+
237+
// Send a coldboot request, confirm it starts.
238+
RebootRequest request;
239+
request.set_method(RebootMethod::COLD);
240+
start_reboot_via_rpc(request);
241+
242+
std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
243+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
244+
RebootBE::RebManagerStatus::COLD_REBOOT_IN_PROGRESS);
245+
246+
// Cleanup without going through the whole reboot.
247+
send_stop_reboot_thread();
248+
force_warm_start_state(false);
249+
}
250+
197251
// Test fixture to skip through the startup sequence into the main loop.
198252
// Param indicates if RebootBE should be initialized into a state where the
199253
// system came up in warmboot.
@@ -203,7 +257,8 @@ class RebootBEAutoStartTest : public RebootBETest,
203257
RebootBEAutoStartTest() {
204258
start_rebootbe();
205259

206-
std::this_thread::sleep_for(std::chrono::milliseconds(50));
260+
// std::this_thread::sleep_for(std::chrono::milliseconds(50));
261+
std::this_thread::sleep_for(std::chrono::milliseconds(ONE_SECOND_MS));
207262
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
208263
}
209264
};
@@ -300,6 +355,30 @@ TEST_P(RebootBEAutoStartTest, TestColdRebootDbusToCompletion) {
300355
"platform failed to reboot"));
301356
}
302357

358+
TEST_P(RebootBEAutoStartTest, TestWarmRebootDbusToCompletion) {
359+
DbusInterface::DbusResponse dbus_response{
360+
DbusInterface::DbusStatus::DBUS_SUCCESS, ""};
361+
EXPECT_CALL(m_dbus_interface, Reboot(_))
362+
.Times(1)
363+
.WillRepeatedly(Return(dbus_response));
364+
365+
overwrite_reboot_timeout(1);
366+
RebootRequest request;
367+
request.set_method(RebootMethod::WARM);
368+
start_reboot_via_rpc(request);
369+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
370+
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);
371+
372+
sleep(TWO_SECONDS);
373+
374+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
375+
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
376+
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
377+
EXPECT_THAT(response,
378+
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
379+
"failed to warm reboot"));
380+
}
381+
303382
TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) {
304383
sigterm_requested = true;
305384
set_mock_defaults();
@@ -319,6 +398,25 @@ TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) {
319398
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
320399
}
321400

401+
TEST_P(RebootBEAutoStartTest, TestWarmBootSigterm) {
402+
sigterm_requested = true;
403+
set_mock_defaults();
404+
overwrite_reboot_timeout(1);
405+
406+
RebootRequest request;
407+
request.set_method(RebootMethod::WARM);
408+
start_reboot_via_rpc(request);
409+
410+
sleep(ONE_SECOND);
411+
412+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
413+
gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc();
414+
EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM));
415+
EXPECT_THAT(
416+
second_resp,
417+
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
418+
}
419+
322420
TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) {
323421
// Return FAIL from dbus reboot call.
324422
DbusInterface::DbusResponse dbus_response{
@@ -341,6 +439,28 @@ TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) {
341439
"dbus reboot failed"));
342440
}
343441

442+
TEST_P(RebootBEAutoStartTest, TestWarmBootDbusError) {
443+
// Return FAIL from dbus reboot call.
444+
DbusInterface::DbusResponse dbus_response{
445+
DbusInterface::DbusStatus::DBUS_FAIL, "dbus reboot failed"};
446+
EXPECT_CALL(m_dbus_interface, Reboot(_))
447+
.Times(1)
448+
.WillOnce(Return(dbus_response));
449+
450+
RebootRequest request;
451+
request.set_method(RebootMethod::WARM);
452+
start_reboot_via_rpc(request);
453+
454+
sleep(TWO_SECONDS);
455+
456+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
457+
gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc();
458+
EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM));
459+
EXPECT_THAT(second_resp,
460+
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
461+
"dbus reboot failed"));
462+
}
463+
344464
TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) {
345465
set_mock_defaults();
346466

@@ -362,12 +482,64 @@ TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) {
362482
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
363483
}
364484

485+
TEST_P(RebootBEAutoStartTest, TestStopDuringWarmBoot) {
486+
set_mock_defaults();
487+
488+
RebootRequest request;
489+
request.set_method(RebootMethod::WARM);
490+
start_reboot_via_rpc(request);
491+
// std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
492+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
493+
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);
494+
495+
send_stop_reboot_thread();
496+
std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
497+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
498+
499+
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
500+
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
501+
EXPECT_THAT(
502+
response,
503+
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
504+
}
505+
365506
TEST_P(RebootBEAutoStartTest, TestInvalidJsonRebootRequest) {
366507
std::string json_request = "abcd";
367508
NotificationResponse response = handle_reboot_request(json_request);
368509
EXPECT_EQ(swss::StatusCode::SWSS_RC_INTERNAL, response.status);
369510
}
370511

512+
TEST_P(RebootBEAutoStartTest, TestWarmFailureFollowedByColdBoot) {
513+
DbusInterface::DbusResponse dbus_response{
514+
DbusInterface::DbusStatus::DBUS_SUCCESS, ""};
515+
overwrite_reboot_timeout(1);
516+
517+
RebootRequest request;
518+
request.set_method(RebootMethod::WARM);
519+
start_reboot_via_rpc(request);
520+
521+
std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
522+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
523+
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);
524+
525+
std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS));
526+
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
527+
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
528+
529+
request.set_method(RebootMethod::COLD);
530+
start_reboot_via_rpc(request);
531+
532+
// We have to wait for the 1 second reboot Timeout
533+
std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS));
534+
535+
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
536+
response = do_reboot_status_rpc();
537+
EXPECT_THAT(response, ActiveCountMethod(false, 2, RebootMethod::COLD));
538+
EXPECT_THAT(response,
539+
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
540+
"platform failed to reboot"));
541+
}
542+
371543
INSTANTIATE_TEST_SUITE_P(TestWithStartupWarmbootEnabledState,
372544
RebootBEAutoStartTest, testing::Values(true, false));
373545

0 commit comments

Comments
 (0)