34#define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
43 inline void Pack(
void* buffer,
int bufferSize,
int*
position , MPI::Intracomm& comm )
const
45 DMSG(4,
"Packing G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
46 <<m_n<<
" "<<m_scale<<
" "<<m_sum_w<<
" "<<m_sum_w2
47 <<
" "<<m_sum_wx<<
" "<<m_sum_wx2);
48 MPI_Pack(&m_n,1,MPI::INT,buffer,bufferSize,
position,comm);
49 const G4double data[]{m_scale,m_sum_w,m_sum_w2,m_sum_wx,m_sum_wx2};
50 MPI_Pack(&data,5,MPI::DOUBLE,buffer,bufferSize,
position,comm);
52 inline void UnPack(
void* buffer,
int bufferSize,
int*
position , MPI::Intracomm& comm ) {
53 MPI_Unpack(buffer,bufferSize,
position,&m_n,1,MPI::INT,comm);
55 MPI_Unpack(buffer,bufferSize,
position,data,5,MPI::DOUBLE,comm);
61 DMSG(4,
"UnPacking G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
62 <<m_n<<
" "<<m_scale<<
" "<<m_sum_w<<
" "<<m_sum_w2
63 <<
" "<<m_sum_wx<<
" "<<m_sum_wx2);
65 MPIStatDouble(G4int ver = 0) : verbose(ver) {}
66 MPIStatDouble(
const G4StatDouble& rhs , G4int ver) : verbose(ver)
68 G4StatDouble::operator=(rhs);
74 outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
75 ownsBuffer(false),scoringManager(nullptr),commSize(0),
82 outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
84 scoringManager(mgr), commSize(0), destinationRank(destination),
124 DMSG(0,
"G4MPIscorerMerger::Merge called");
128 DMSG(1,
"Comm world size is 1, nothing to do");
132 comm = parentComm->Dup();
151 const G4double sttime = MPI::Wtime();
154 typedef std::function<void(
unsigned int)> handler_t;
155 using std::placeholders::_1;
158 std::function<void(
void)> barrier = std::bind(&MPI::Intracomm::Barrier,&
comm);
177 const G4double elapsed = MPI::Wtime() - sttime;
182 G4cout<<
"G4MPIscorerMerger::Merge() -data transfer performances: "
183 <<double(total)/1000./elapsed<<
" kB/s"
184 <<
" (Total Data Transfer= "<<double(total)/1000.<<
" kB in "
185 <<elapsed<<
" s)."<<G4endl;
202 DMSG(0,
"G4MPIscorerMerger::Merge done.");
206 DMSG(1,
"Receiving scorers");
208 DMSG(2,
"Receiving from: "<<source);
211 const G4int newbuffsize = status.Get_count(MPI::PACKED);
212 DMSG(2,
"Preparing to receive buffer of size: "<<newbuffsize);
215 DMSG(3,
"New larger buffer expected, resize");
219 buffer =
new char[newbuffsize];
222 std::fill( buffer , buffer + newbuffsize , 0 );
226 comm.Recv(buffer, newbuffsize, MPI::PACKED, source,
230 DMSG(1,
"Receiving of comamnd line scorers done");
234 DMSG(1,
"Sending scorers "<<
this);
241 buffer =
new char[newbuffsize];
244 std::fill( buffer , buffer+newbuffsize,0);
256 DMSG(1,
"Sending done");
262 G4Exception(
"G4MPIscorerMerger::Pack(const G4ScoringManager*)",
263 "MPI001",FatalException,
264 "Call SetOututBuffer before trying to pack");
267 DMSG(2,
"Starting packing of meshes, # meshes: "<<sm->GetNumberOfMesh());
268 size_t numMeshes=sm->GetNumberOfMesh();
269 MPI_Pack(&numMeshes,1,MPI::UNSIGNED,
273 for (
size_t i = 0; i <numMeshes; ++i)
275 MPI_Pack(&i,1,MPI::UNSIGNED,
278 Pack(sm->GetMesh(i));
285 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
286 "MPI001",FatalException,
287 "Call SetOututBuffer before trying to un-pack");
292 &numMeshes,1,MPI::UNSIGNED,
comm);
293 if ( numMeshes != sm->GetNumberOfMesh() ) {
294 G4ExceptionDescription msg;
295 msg <<
"Number of meshes to unpack ("<<numMeshes;
296 msg <<
") does not correspond to expected number ("<<sm->GetNumberOfMesh();
298 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
299 "MPI001",FatalException,msg);
304 for (
size_t i = 0 ; i < numMeshes ; ++i ) {
306 &meshid,1,MPI::UNSIGNED,
comm);
308 G4ExceptionDescription msg;
309 msg<<
"Cannot unpack: expecting mesh "<<i<<
" and found "<<meshid;
310 msg<<
" during unpack.";
311 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
312 "MPI001",FatalException,msg);
321 assert(mesh!=
nullptr);
324 DMSG(3,
"Packing mesh: "<<mesh);
326 auto map = mesh->GetScoreMap();
327 size_t nummaps = map.size();
328 MPI_Pack(&nummaps,1,MPI::UNSIGNED,
331 for (
const auto& ele: map ) {
333 size_t ss = name.size();
334 MPI_Pack(&ss,1,MPI::UNSIGNED,
337#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
338 char* nn =
new char[name.length()];
339 std::copy(name.begin(),name.end(),nn);
341 const char* nn = name.c_str();
345#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
354 assert(inmesh!=
nullptr);
355 DMSG(3,
"Preparing to unpack a mesh and merge into: "<<inmesh);
356 const G4String& detName = inmesh->GetWorldName();
359 &nummaps,1,MPI::UNSIGNED,
comm);
360 for (
size_t i = 0 ; i < nummaps ; ++i ) {
363 &nameSize,1,MPI::UNSIGNED,
comm);
366 char* name =
new char[nameSize+1];
367 std::fill(name,name+nameSize+1,0);
369 name,nameSize,MPI::CHAR,
comm);
370 const G4String colname(name,nameSize);
377 inmesh->Accumulate(hm);
412 DMSG(3,
"Packing hitmap: "<<sm<<
" with: "<<sm->GetSize()<<
" elements.");
413 size_t numEl = sm->GetSize();
414 MPI_Pack(&numEl,1,MPI::UNSIGNED,
417 const auto& theMap = *sm->GetMap();
418 std::vector<G4int> ids;
419 std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
420 [](
decltype(*theMap.begin())& e){ return e.first;});
421 assert(ids.size()==numEl);
424 for(
const auto& e : theMap) {
425 const MPIStatDouble sd(*e.second,
verbose);
456 DMSG(3,
"Preparing to unpack a hit map for: "<<detName<<
","<<colName);
459 &numEl,1,MPI::UNSIGNED,
comm);
460 DMSG(3,
"Will receive "<<numEl<<
" values");
461 G4int* ids =
new G4int[numEl];
463 ids,numEl,MPI::INT,
comm);
465 for (
unsigned int i = 0; i<numEl;++i) {
468 result->set(ids[i],sd);
477 DMSG(3,
"Calculating dimension of data to send");
478 if ( sm ==
nullptr )
return 0;
484 G4int size =
sizeof(
unsigned int);
485 DMSG(3,
"There are "<<sm->GetNumberOfMesh()<<
" meshes.");
487 for (
size_t i = 0 ; i<sm->GetNumberOfMesh() ; ++i ) {
488 size +=
sizeof(
unsigned int);
496 DMSG(3,
"Calculating size for mesh: "<<mesh);
498 G4int size =
sizeof(
unsigned int);
499 auto map = mesh->GetScoreMap();
500 for (
const auto& ele : map ) {
502 size +=
sizeof(
unsigned int);
504 size +=
sizeof(char)*name.size();
507 DMSG(3,
"mesh "<<mesh<<
" size: "<<size);
521 const G4int numEls = map->GetSize();
522 G4int size =
sizeof(
unsigned int);
523 size +=
sizeof(G4int)*numEls;
526 size += numEls*(
sizeof(G4double)*5+
sizeof(G4int));
527 DMSG(3,
"HitStatDoubleMap "<<map<<
" size: "<<size<<
" in "<<numEls<<
" elements.");
G4THitsMap< G4StatDouble > HitStatDoubleMap
static G4MPImanager * GetManager()
G4int GetActiveSize() const
const MPI::Intracomm * GetComm() const
G4int CalculatePackSize(const G4ScoringManager *) const
void Send(const unsigned int destination)
unsigned int destinationRank
void Receive(const unsigned int source)
G4ScoringManager * scoringManager
HitStatDoubleMap * UnPackHitStatDoubleMap(const G4String &detName, const G4String &colName)
void SetupOutputBuffer(char *buff, G4int size, G4int position)
void Pack(const G4ScoringManager *)
Pack all meshes into buffer.
virtual ~G4MPIscorerMerger()
G4int outputBufferPosition
void UnPackAndMerge(const G4ScoringManager *)
void Merge(std::function< void(unsigned int)> senderF, std::function< void(unsigned int)> receiverF, std::function< void(void)> barrierF, unsigned int commSize, unsigned int myrank)