From d556da25a48f2c944845a28cb7020d1179a3dfa2 Mon Sep 17 00:00:00 2001 From: Dennis Terhorst Date: Fri, 2 Feb 2024 13:19:38 +0100 Subject: [PATCH 1/4] change omp single to master MPI calls have to be done from the *same* thread (`master`) instead of from a `single` thread in MPI_FUNNELED mode according to the specifications. --- nestkernel/event_delivery_manager.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/nestkernel/event_delivery_manager.cpp b/nestkernel/event_delivery_manager.cpp index a568df519c..e6fa11a451 100644 --- a/nestkernel/event_delivery_manager.cpp +++ b/nestkernel/event_delivery_manager.cpp @@ -804,13 +804,14 @@ EventDeliveryManager::gather_target_data( const size_t tid ) // otherwise gather_completed_checker_[ tid ].set_true(); -#pragma omp single +#pragma omp master { if ( kernel().mpi_manager.adaptive_target_buffers() and buffer_size_target_data_has_changed_ ) { resize_send_recv_buffers_target_data(); } - } // of omp single; implicit barrier + } // of omp master; (no barrier) + #pragma omp barrier kernel().connection_manager.restore_source_table_entry_point( tid ); @@ -828,7 +829,7 @@ EventDeliveryManager::gather_target_data( const size_t tid ) #pragma omp barrier kernel().connection_manager.clean_source_table( tid ); -#pragma omp single +#pragma omp master { #ifdef TIMER_DETAILED sw_communicate_target_data_.start(); @@ -837,7 +838,7 @@ EventDeliveryManager::gather_target_data( const size_t tid ) #ifdef TIMER_DETAILED sw_communicate_target_data_.stop(); #endif - } // of omp single (implicit barrier) + } // of omp master (no barriers!) const bool distribute_completed = distribute_target_data_buffers_( tid ); @@ -846,7 +847,7 @@ EventDeliveryManager::gather_target_data( const size_t tid ) // resize mpi buffers, if necessary and allowed if ( gather_completed_checker_.any_false() and kernel().mpi_manager.adaptive_target_buffers() ) { -#pragma omp single +#pragma omp master { buffer_size_target_data_has_changed_ = kernel().mpi_manager.increase_buffer_size_target_data(); } @@ -874,13 +875,14 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid ) // assume this is the last gather round and change to false otherwise gather_completed_checker_[ tid ].set_true(); -#pragma omp single +#pragma omp master { if ( kernel().mpi_manager.adaptive_target_buffers() and buffer_size_target_data_has_changed_ ) { resize_send_recv_buffers_target_data(); } - } // of omp single; implicit barrier + } // of omp master; no barrier +#pragma omp barrier TargetSendBufferPosition send_buffer_position( assigned_ranks, kernel().mpi_manager.get_send_recv_count_target_data_per_rank() ); @@ -897,7 +899,7 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid ) #pragma omp barrier -#pragma omp single +#pragma omp master { #ifdef TIMER_DETAILED sw_communicate_target_data_.start(); @@ -906,7 +908,8 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid ) #ifdef TIMER_DETAILED sw_communicate_target_data_.stop(); #endif - } // of omp single (implicit barrier) + } // of omp master (no barrier) +#pragma omp barrier // Up to here, gather_completed_checker_ just has local info: has this thread been able to write // all data it is responsible for to buffers. Now combine with information on whether other ranks @@ -917,7 +920,7 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid ) // resize mpi buffers, if necessary and allowed if ( gather_completed_checker_.any_false() and kernel().mpi_manager.adaptive_target_buffers() ) { -#pragma omp single +#pragma omp master { buffer_size_target_data_has_changed_ = kernel().mpi_manager.increase_buffer_size_target_data(); } From 1ef3f6444959ba193e420c489d1a33e1aee872d5 Mon Sep 17 00:00:00 2001 From: Dennis Terhorst Date: Fri, 2 Feb 2024 13:48:27 +0100 Subject: [PATCH 2/4] typo --- nestkernel/event_delivery_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nestkernel/event_delivery_manager.cpp b/nestkernel/event_delivery_manager.cpp index e6fa11a451..e2b2233a65 100644 --- a/nestkernel/event_delivery_manager.cpp +++ b/nestkernel/event_delivery_manager.cpp @@ -804,14 +804,14 @@ EventDeliveryManager::gather_target_data( const size_t tid ) // otherwise gather_completed_checker_[ tid ].set_true(); -#pragma omp master +#pragma omp { if ( kernel().mpi_manager.adaptive_target_buffers() and buffer_size_target_data_has_changed_ ) { resize_send_recv_buffers_target_data(); } } // of omp master; (no barrier) - #pragma omp barrier +#pragma omp barrier kernel().connection_manager.restore_source_table_entry_point( tid ); From 5786b73535618751e33ec834ad392abf086f721d Mon Sep 17 00:00:00 2001 From: Dennis Terhorst Date: Fri, 2 Feb 2024 13:50:23 +0100 Subject: [PATCH 3/4] re-add removed barrier --- nestkernel/event_delivery_manager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nestkernel/event_delivery_manager.cpp b/nestkernel/event_delivery_manager.cpp index e2b2233a65..b4e5af6d60 100644 --- a/nestkernel/event_delivery_manager.cpp +++ b/nestkernel/event_delivery_manager.cpp @@ -923,7 +923,8 @@ EventDeliveryManager::gather_target_data_compressed( const size_t tid ) #pragma omp master { buffer_size_target_data_has_changed_ = kernel().mpi_manager.increase_buffer_size_target_data(); - } + } // of omp master (no barrier) +#pragma omp barrier } } // of while From 8f3c3036b439b4731529805dc7adcc72975d45e2 Mon Sep 17 00:00:00 2001 From: Dennis Terhorst Date: Fri, 2 Feb 2024 16:28:59 +0100 Subject: [PATCH 4/4] add barriers at end of previous single sections --- nestkernel/event_delivery_manager.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nestkernel/event_delivery_manager.cpp b/nestkernel/event_delivery_manager.cpp index b4e5af6d60..4979b10b5e 100644 --- a/nestkernel/event_delivery_manager.cpp +++ b/nestkernel/event_delivery_manager.cpp @@ -804,7 +804,7 @@ EventDeliveryManager::gather_target_data( const size_t tid ) // otherwise gather_completed_checker_[ tid ].set_true(); -#pragma omp +#pragma omp master { if ( kernel().mpi_manager.adaptive_target_buffers() and buffer_size_target_data_has_changed_ ) { @@ -839,7 +839,7 @@ EventDeliveryManager::gather_target_data( const size_t tid ) sw_communicate_target_data_.stop(); #endif } // of omp master (no barriers!) - +#pragma omp barrier const bool distribute_completed = distribute_target_data_buffers_( tid ); gather_completed_checker_[ tid ].logical_and( distribute_completed ); @@ -851,6 +851,7 @@ EventDeliveryManager::gather_target_data( const size_t tid ) { buffer_size_target_data_has_changed_ = kernel().mpi_manager.increase_buffer_size_target_data(); } +#pragma omp barrier } } // of while