1#ifndef DUNE_SPGRID_COMMUNICATION_HH
2#define DUNE_SPGRID_COMMUNICATION_HH
4#include <dune/common/hybridutilities.hh>
5#include <dune/common/parallel/communication.hh>
6#include <dune/common/parallel/mpicommunication.hh>
7#include <dune/common/parallel/mpitraits.hh>
8#include <dune/common/visibility.hh>
10#include <dune/grid/common/exceptions.hh>
11#include <dune/grid/common/datahandleif.hh>
22 template<
class Comm >
52 return comm( MPI_COMM_WORLD );
64 static unsigned char counter = 0;
65 return int( counter++ ) + 1536;
75 template<
class Gr
id,
class DataHandle >
83 typedef typename DataHandle::DataType
DataType;
93 InterfaceType iftype, CommunicationDirection dir );
100 bool ready ()
const {
return !( bool( interface_ )); }
108 DataHandle &dataHandle_;
110 CommunicationDirection dir_;
113 std::vector< WriteBuffer > writeBuffers_;
114 std::vector< ReadBuffer > readBuffers_;
122 template<
class Gr
id,
class DataHandle >
123 inline SPCommunication< Grid, DataHandle >
124 ::SPCommunication (
const GridLevel &gridLevel, DataHandle &dataHandle,
125 InterfaceType iftype, CommunicationDirection dir )
126 : gridLevel_( gridLevel ),
127 dataHandle_( dataHandle ),
128 interface_( &gridLevel.commInterface( iftype ) ),
130 tag_( __SPGrid::getCommTag() ),
133 for(
int codim = 0; codim <=
dimension; ++codim )
134 fixedSize_ &= !dataHandle_.contains(
dimension, codim ) || dataHandle_.fixedSize(
dimension, codim );
136 const std::size_t numLinks = interface_->
size();
137 readBuffers_.reserve( numLinks );
143 readBuffers_.emplace_back( gridLevel.
grid().comm() );
144 std::size_t size = 0;
146 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [
this, &partitionList, &size ] (
auto codim ) {
149 if( !dataHandle_.contains(
dimension, codim ) )
152 const Iterator end( gridLevel_, partitionList,
typename Iterator::End() );
153 for( Iterator it( gridLevel_, partitionList,
typename Iterator::Begin() ); it != end; ++it )
154 size += dataHandle_.size( *it );
157 readBuffers_.back().receive( it->rank(), tag_, size );
161 writeBuffers_.reserve( numLinks );
164 writeBuffers_.emplace_back( gridLevel.
grid().comm() );
166 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [
this, &partitionList ] (
auto codim ) {
169 if( !dataHandle_.contains(
dimension, codim ) )
172 const bool fixedSize = dataHandle_.fixedSize(
dimension, codim );
173 const Iterator end( gridLevel_, partitionList,
typename Iterator::End() );
174 for( Iterator it( gridLevel_, partitionList,
typename Iterator::Begin() ); it != end; ++it )
176 const auto &entity = *it;
178 writeBuffers_.back().write(
static_cast< int >( dataHandle_.size( entity ) ) );
180 const std::size_t posBeforeGather = writeBuffers_.back().position();
182 dataHandle_.gather( writeBuffers_.back(), entity );
184 const std::size_t posAfterGather = writeBuffers_.back().position();
185 const std::size_t sizeInBytes = dataHandle_.size( entity ) *
sizeof(
DataType );
186 if( posAfterGather - posBeforeGather != sizeInBytes )
187 DUNE_THROW( GridError,
"Number of bytes written (" << (posAfterGather - posBeforeGather) <<
") does not coincide with reported size (" << sizeInBytes <<
")" );
191 writeBuffers_.back().send( it->rank(), tag_ );
196 template<
class Gr
id,
class DataHandle >
198 : gridLevel_( other.gridLevel_ ),
199 dataHandle_( other.dataHandle_ ),
200 interface_( other.interface_ ),
203 fixedSize_( other.fixedSize_ ),
204 writeBuffers_(
std::move( other.writeBuffers_ ) ),
205 readBuffers_(
std::move( other.readBuffers_ ) )
207 other.interface_ =
nullptr;
211 template<
class Gr
id,
class DataHandle >
217 const std::size_t numLinks = interface_->size();
221 for( std::size_t i = 0; i < numLinks; ++i )
223 readBuffers_.emplace_back( gridLevel_.grid().comm() );
224 readBuffers_.back().receive( tag_ );
228 for( std::size_t i = 0; i < numLinks; ++i )
230 const typename std::vector< ReadBuffer >::iterator buffer = waitAny( readBuffers_ );
231 for(
typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
233 if( it->rank() == buffer->rank() )
235 const PartitionList &partitionList = it->receiveList( dir_ );
236 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [
this, &partitionList, buffer ] (
auto codim ) {
239 if( !dataHandle_.contains( dimension, codim ) )
242 const bool fixedSize = dataHandle_.fixedSize( dimension, codim );
243 const Iterator end( gridLevel_, partitionList,
typename Iterator::End() );
244 for( Iterator it( gridLevel_, partitionList,
typename Iterator::Begin() ); it != end; ++it )
246 const auto &entity = *it;
250 buffer->read( size );
252 size = dataHandle_.size( entity );
254 const std::size_t posBeforeGather = buffer->position();
256 dataHandle_.scatter( *buffer, entity, size );
258 const std::size_t posAfterGather = buffer->position();
259 const std::size_t sizeInBytes =
static_cast< std::size_t
>( size ) *
sizeof(
DataType );
260 if( posAfterGather - posBeforeGather != sizeInBytes )
261 DUNE_THROW( GridError,
"Number of bytes read (" << (posAfterGather - posBeforeGather) <<
") does not coincide with reported size (" << sizeInBytes <<
")" );
269 readBuffers_.clear();
271 for(
typename std::vector< WriteBuffer >::iterator it = writeBuffers_.begin(); it != writeBuffers_.end(); ++it )
273 writeBuffers_.clear();
275 interface_ =
nullptr;
Definition: iostream.hh:7
DUNE_EXPORT int getCommTag()
Definition: communication.hh:62
Definition: communication.hh:24
Dune::CollectiveCommunication< Comm > CollectiveCommunication
Definition: communication.hh:25
static CollectiveCommunication comm(const C &)
Definition: communication.hh:28
static CollectiveCommunication defaultComm()
Definition: communication.hh:33
static CollectiveCommunication defaultComm()
Definition: communication.hh:50
static CollectiveCommunication comm(const MPI_Comm &mpiComm)
Definition: communication.hh:45
Dune::CollectiveCommunication< MPI_Comm > CollectiveCommunication
Definition: communication.hh:43
Definition: communication.hh:77
static const int dimension
Definition: communication.hh:78
bool pending() const
Definition: communication.hh:105
DataHandle::DataType DataType
Definition: communication.hh:83
GridLevel::CommInterface Interface
Definition: communication.hh:85
SPCommunication(const SPCommunication &)=delete
void wait()
Definition: communication.hh:212
SPCommunication(const GridLevel &gridLevel, DataHandle &dataHandle, InterfaceType iftype, CommunicationDirection dir)
Definition: communication.hh:124
SPPartitionList< dimension > PartitionList
Definition: communication.hh:81
bool ready() const
Definition: communication.hh:100
SPGridLevel< Grid > GridLevel
Definition: communication.hh:80
~SPCommunication()
Definition: communication.hh:98
Definition: gridlevel.hh:35
const Grid & grid() const
Definition: gridlevel.hh:82
Definition: iterator.hh:19
Definition: linkage.hh:54
NodeContainer::const_iterator Iterator
Definition: linkage.hh:60
Iterator end() const
Definition: linkage.hh:68
Iterator begin() const
Definition: linkage.hh:67
std::size_t size() const
Definition: linkage.hh:70
Definition: messagebuffer.hh:91
Definition: messagebuffer.hh:204
Definition: partitionlist.hh:16