G4MPIscorerMerger Class Reference

#include <Doxymodules_parallel.h>

Public Member Functions
	G4MPIscorerMerger ()
	G4MPIscorerMerger (G4ScoringManager *mgr, G4int destination=G4MPImanager::kRANK_MASTER, G4int verbosity=0)
virtual	~G4MPIscorerMerger ()
void	SetDestinationRank (G4int i)
void	SetScoringManager (G4ScoringManager *mgr)
void	SetVerbosity (G4int ver)
void	Merge ()

Protected Member Functions
void	SetupOutputBuffer (char *buff, G4int size, G4int position)
void	DestroyBuffer ()
void	Pack (const G4ScoringManager *)
	Pack all meshes into buffer.
void	UnPackAndMerge (const G4ScoringManager *)
void	Pack (const G4VScoringMesh *)
	Pack a single mesh.
void	UnPackAndMerge (G4VScoringMesh *)
void	Pack (const HitStatDoubleMap *)
	Pack a single score map.
HitStatDoubleMap *	UnPackHitStatDoubleMap (const G4String &detName, const G4String &colName)
G4int	CalculatePackSize (const G4ScoringManager *) const
G4int	CalculatePackSize (const G4VScoringMesh *) const
G4int	CalculatePackSize (const HitStatDoubleMap *) const
void	Send (const unsigned int destination)
void	Receive (const unsigned int source)

Private Attributes
char *	outputBuffer
G4int	outputBufferSize
G4int	outputBufferPosition
long	bytesSent
G4bool	ownsBuffer
G4ScoringManager *	scoringManager
unsigned int	commSize
unsigned int	destinationRank
MPI::Intracomm	comm
G4int	verbose

Detailed Description

Definition at line 67 of file Doxymodules_parallel.h.

Constructor & Destructor Documentation

G4MPIscorerMerger() [1/2]

G4MPIscorerMerger::G4MPIscorerMerger

(

)

Definition at line 73 of file G4MPIscorerMerger.cc.

                                     :
  outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
  ownsBuffer(false),scoringManager(nullptr),commSize(0),
  destinationRank(G4MPImanager::kRANK_MASTER),verbose(0)
{}

G4MPIscorerMerger() [2/2]

G4MPIscorerMerger::G4MPIscorerMerger	(	G4ScoringManager *	mgr,
		G4int	destination = G4MPImanager::kRANK_MASTER,
		G4int	verbosity = 0
	)

Definition at line 79 of file G4MPIscorerMerger.cc.

                     :
    outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
    ownsBuffer(false),
  scoringManager(mgr), commSize(0), destinationRank(destination),
      verbose(verbosity)
{
}

~G4MPIscorerMerger()

G4MPIscorerMerger::~G4MPIscorerMerger ( )

virtual

Definition at line 89 of file G4MPIscorerMerger.cc.

                                      {
  if ( ownsBuffer ) delete[] outputBuffer;
}

Member Function Documentation

SetDestinationRank()

void G4MPIscorerMerger::SetDestinationRank ( G4int  i )

inline

Definition at line 49 of file G4MPIscorerMerger.hh.

{ destinationRank = i; }

SetScoringManager()

void G4MPIscorerMerger::SetScoringManager ( G4ScoringManager *  mgr )

inline

Definition at line 50 of file G4MPIscorerMerger.hh.

{ scoringManager = mgr; }

SetVerbosity()

void G4MPIscorerMerger::SetVerbosity ( G4int  ver )

inline

Definition at line 51 of file G4MPIscorerMerger.hh.

{ verbose = ver; }

Merge()

void G4MPIscorerMerger::Merge

(

)

Definition at line 123 of file G4MPIscorerMerger.cc.

                              {
  DMSG(0, "G4MPIscorerMerger::Merge called");
  const unsigned int myrank = G4MPImanager::GetManager()->GetRank();
  commSize = G4MPImanager::GetManager()->GetActiveSize();
  if ( commSize == 1 ) {
      DMSG(1,"Comm world size is 1, nothing to do");
      return;
  }
  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
  comm = parentComm->Dup();
  DestroyBuffer();
 
  //ANDREA:->
//  G4cout<<"Before sending: "<<G4endl;
//  scoringManager->GetMesh(0)->Dump();
//  for ( int i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i )  {
//  for ( auto e : scoringManager->GetMesh(i)->GetScoreMap() )
//    {
//      G4cout<<e.first<<" : "<<e.second<<G4endl;
//      for ( auto c: *(e.second->GetMap()) ) {
//          G4cout<<c.first<<"="<<*c.second<<G4endl;
//
//     }
//    }
//  }
  //ANDREA:<-
 
  bytesSent=0;
  const G4double sttime = MPI::Wtime();
 
  //Use G4MPIutils to optimize communications between ranks
  typedef std::function<void(unsigned int)> handler_t;
  using std::placeholders::_1;
  handler_t sender = std::bind(&G4MPIscorerMerger::Send , this , _1);
  handler_t receiver = std::bind(&G4MPIscorerMerger::Receive, this, _1);
  std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier,&comm);
  G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
 
  //OLD Style p2p communications
  /*
  if ( myrank != destinationRank ) {
      DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
          <<myrank<<" sending to rank "<<destinationRank
          <<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
      Send(destinationRank);
  } else {
      DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
          <<myrank<<" receiving "
          <<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
      for ( unsigned int i = 0 ; i < commSize ; ++i ) {
          if ( i != myrank ) Receive(i);
      }
  }
*/
  const G4double elapsed = MPI::Wtime() - sttime;
  long total=0;
  comm.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,destinationRank);
  if ( verbose > 0 && myrank == destinationRank ) {
      //Collect from ranks how much data was sent around
      G4cout<<"G4MPIscorerMerger::Merge() -data transfer performances: "
                  <<double(total)/1000./elapsed<<" kB/s"
                  <<" (Total Data Transfer= "<<double(total)/1000.<<" kB in "
                  <<elapsed<<" s)."<<G4endl;
  }
  //ANDREA:->
//  G4cout<<"After Receiving: "<<G4endl;
//  scoringManager->GetMesh(0)->Dump();
//  for ( int i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i )  {
//  for ( auto e : scoringManager->GetMesh(i)->GetScoreMap() )
//    {
//      G4cout<<e.first<<" : "<<e.second<<G4endl;
//      for ( auto c: *(e.second->GetMap()) ) {
//          G4cout<<c.first<<"="<<*c.second<<" (=2x"<<.5*(*c.second)<<")"<<G4endl;
//
//     }
//    }
//  }
  //ANDREA:<-
  comm.Free();
  DMSG(0,"G4MPIscorerMerger::Merge done.");
}

SetupOutputBuffer()

void G4MPIscorerMerger::SetupOutputBuffer ( char *  buff, G4int  size, G4int  position  )

inlineprotected

Definition at line 57 of file G4MPIscorerMerger.hh.

                                                                 {
    outputBuffer = buff;
    outputBufferSize=size;
    outputBufferPosition=position;
  }

DestroyBuffer()

void G4MPIscorerMerger::DestroyBuffer ( )

inlineprotected

Definition at line 62 of file G4MPIscorerMerger.hh.

                       {
    delete[] outputBuffer;
    outputBuffer = nullptr;
    outputBufferSize=0;
    outputBufferPosition=0;
    ownsBuffer = false;
  }

Pack() [1/3]

void G4MPIscorerMerger::Pack ( const G4ScoringManager *  sm )

protected

Pack all meshes into buffer.

Definition at line 259 of file G4MPIscorerMerger.cc.

                                                       {
  assert(sm!=nullptr);
  if ( outputBuffer == nullptr || outputBufferPosition>=outputBufferSize) {
      G4Exception("G4MPIscorerMerger::Pack(const G4ScoringManager*)",
                "MPI001",FatalException,
                "Call SetOututBuffer before trying to pack");
      return;
  }
  DMSG(2,"Starting packing of meshes, # meshes: "<<sm->GetNumberOfMesh());
  /*const*/ size_t numMeshes=sm->GetNumberOfMesh();//TODO: OLD MPI interface
  MPI_Pack(&numMeshes,1,MPI::UNSIGNED,
                outputBuffer,outputBufferSize,
                &outputBufferPosition,
                comm);
  for (size_t i = 0; i <numMeshes; ++i)
    {
      MPI_Pack(&i,1,MPI::UNSIGNED,
          outputBuffer,outputBufferSize,
          &outputBufferPosition,comm);
      Pack(sm->GetMesh(i));
    }
}

UnPackAndMerge() [1/2]

void G4MPIscorerMerger::UnPackAndMerge ( const G4ScoringManager *  sm )

protected

Definition at line 282 of file G4MPIscorerMerger.cc.

                                                                 {
  assert(sm!=nullptr);
  if ( outputBuffer == nullptr || outputBufferPosition>=outputBufferSize) {
      G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
                "MPI001",FatalException,
                "Call SetOututBuffer before trying to un-pack");
      return;
  }
  size_t numMeshes=0;
  MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
      &numMeshes,1,MPI::UNSIGNED,comm);
  if ( numMeshes != sm->GetNumberOfMesh() ) {
      G4ExceptionDescription msg;
      msg << "Number of meshes to unpack ("<<numMeshes;
      msg <<") does not correspond to expected number ("<<sm->GetNumberOfMesh();
      msg<<")";
      G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
                "MPI001",FatalException,msg);
      return;
  }
 
  size_t meshid=0;
  for ( size_t i = 0 ; i < numMeshes ; ++i ) {
      MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
          &meshid,1,MPI::UNSIGNED,comm);
      if ( meshid != i )  {
          G4ExceptionDescription msg;
          msg<<"Cannot unpack: expecting mesh "<<i<<" and found "<<meshid;
          msg<<" during unpack.";
          G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
                    "MPI001",FatalException,msg);
          return;
      }
      G4VScoringMesh* original = sm->GetMesh(i);
      UnPackAndMerge(original);
  }
}

Pack() [2/3]

void G4MPIscorerMerger::Pack ( const G4VScoringMesh *  mesh )

protected

Pack a single mesh.

Definition at line 320 of file G4MPIscorerMerger.cc.

                                                       {
  assert(mesh!=nullptr);
  assert(outputBuffer!=nullptr);
  assert(outputBufferPosition<=outputBufferSize);
  DMSG(3,"Packing mesh: "<<mesh);
 
  auto map = mesh->GetScoreMap();
  /*const*/ size_t nummaps = map.size();//TODO: old MPI interface
  MPI_Pack(&nummaps,1,MPI::UNSIGNED,
      outputBuffer,outputBufferSize,
      &outputBufferPosition,comm);
  for ( const auto& ele: map ) {
      const G4String& name = ele.first;
      /*const*/ size_t ss = name.size();
      MPI_Pack(&ss,1,MPI::UNSIGNED,
          outputBuffer,outputBufferSize,
          &outputBufferPosition,comm);
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
      char* nn = new char[name.length()];
      std::copy(name.begin(),name.end(),nn); 
#else
      const char* nn = name.c_str();
#endif
      MPI_Pack(nn,ss,MPI::CHAR,outputBuffer,outputBufferSize,&outputBufferPosition,comm);
      Pack(ele.second);
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
      delete[] nn;
#endif
  }
}

UnPackAndMerge() [2/2]

void G4MPIscorerMerger::UnPackAndMerge ( G4VScoringMesh *  inmesh )

protected

Definition at line 351 of file G4MPIscorerMerger.cc.

                                                             {
  assert(outputBuffer!=nullptr);
  assert(outputBufferPosition<=outputBufferSize);
  assert(inmesh!=nullptr);
  DMSG(3,"Preparing to unpack a mesh and merge into: "<<inmesh);
  const G4String& detName = inmesh->GetWorldName();
  size_t nummaps = 0;
  MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
      &nummaps,1,MPI::UNSIGNED,comm);
  for ( size_t i = 0 ; i < nummaps ; ++i ) {
      size_t nameSize = 0;
      MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
          &nameSize,1,MPI::UNSIGNED,comm);
      //Create a null-terminated c-string: needed later when converting this to a G4String
      //(Not sure: but issue reported by valgrind with the use of MPI_Unpack)
      char* name = new char[nameSize+1];
      std::fill(name,name+nameSize+1,0);
      MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
          name,nameSize,MPI::CHAR,comm);
      const G4String colname(name,nameSize);
      delete[] name;
      //This memory churn is very inefficient, but we cannot reuse the HitMap
      //because we cannot change the names
      //TODO: Evaluate change in HitMap class to allow for change of names
      //HitMap* hm = UnPackHitMap(detName,colname);
      HitStatDoubleMap* hm = UnPackHitStatDoubleMap(detName,colname);
      inmesh->Accumulate(hm);
      delete hm;
  }
}

Pack() [3/3]

void G4MPIscorerMerger::Pack ( const HitStatDoubleMap *  sm )

protected

Pack a single score map.

Definition at line 408 of file G4MPIscorerMerger.cc.

                                                       {
  assert(sm!=nullptr);
  assert(outputBuffer!=nullptr);
  assert(outputBufferPosition<=outputBufferSize);
  DMSG(3,"Packing hitmap: "<<sm<<" with: "<<sm->GetSize()<<" elements.");
  /*const*/ size_t numEl = sm->GetSize();//TODO: old MPI implementation
  MPI_Pack(&numEl,1,MPI::UNSIGNED,
      outputBuffer,outputBufferSize,
      &outputBufferPosition,comm);
  const auto& theMap = *sm->GetMap();
  std::vector<G4int> ids;
  std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
      [](decltype(*theMap.begin())& e){ return e.first;});
  assert(/*ids.size()==vals.size()&&*/ids.size()==numEl);
  MPI_Pack(ids.data(),ids.size(),MPI::INT,outputBuffer,outputBufferSize,
      &outputBufferPosition,comm);
  for( const auto& e : theMap) {
     const MPIStatDouble sd(*e.second,verbose);
     sd.Pack(outputBuffer,outputBufferSize,&outputBufferPosition,comm);
  }
}

UnPackHitStatDoubleMap()

HitStatDoubleMap * G4MPIscorerMerger::UnPackHitStatDoubleMap ( const G4String &  detName, const G4String &  colName  )

protected

Definition at line 451 of file G4MPIscorerMerger.cc.

{
  assert(outputBuffer!=nullptr);
  assert(outputBufferPosition<=outputBufferSize);
  DMSG(3,"Preparing to unpack a hit map for: "<<detName<<","<<colName);
  size_t numEl =0 ;
  MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
             &numEl,1,MPI::UNSIGNED,comm);
  DMSG(3,"Will receive "<<numEl<<" values");
  G4int* ids = new G4int[numEl];
  MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
             ids,numEl,MPI::INT,comm);
  HitStatDoubleMap* result = new HitStatDoubleMap(detName,colName);
  for ( unsigned int i = 0; i<numEl;++i) {
      MPIStatDouble sd(verbose);
      sd.UnPack(outputBuffer,outputBufferSize,&outputBufferPosition,comm);
      result->set(ids[i],sd);
    }
  delete[] ids;
  return result;
}

CalculatePackSize() [1/3]

G4int G4MPIscorerMerger::CalculatePackSize ( const G4ScoringManager *  sm ) const

protected

Definition at line 475 of file G4MPIscorerMerger.cc.

{
  DMSG(3,"Calculating dimension of data to send");
  if ( sm == nullptr ) return 0;
  //Calcualte how much data each call to Pack* appends to the buffer
  //e.g. sizeof(data)
  //The number of sizeof here should match the number of calls to MPI_Pack
 
  //Pack(ScoringMgr)
  G4int size = sizeof(unsigned int);
  DMSG(3,"There are "<<sm->GetNumberOfMesh()<<" meshes.");
  //Loop on mesh
  for ( size_t i = 0 ; i<sm->GetNumberOfMesh() ; ++i ) {
      size += sizeof(unsigned int);//ID
      size += CalculatePackSize(sm->GetMesh(i));
  }
  return size;
}

CalculatePackSize() [2/3]

G4int G4MPIscorerMerger::CalculatePackSize ( const G4VScoringMesh *  mesh ) const

protected

Definition at line 494 of file G4MPIscorerMerger.cc.

{
  DMSG(3,"Calculating size for mesh: "<<mesh);
  //PackSingleMesh(Mesh)
  G4int size = sizeof(unsigned int);//num maps
  auto map = mesh->GetScoreMap();
  for (const auto& ele : map ) {
      //PackHitsMap
      size += sizeof(unsigned int);//name size
      const G4String& name = ele.first;
      size += sizeof(char)*name.size();//name
      size += CalculatePackSize(ele.second);
  }
  DMSG(3,"mesh "<<mesh<<" size: "<<size);
  return size;
}

CalculatePackSize() [3/3]

G4int G4MPIscorerMerger::CalculatePackSize ( const HitStatDoubleMap *  map ) const

protected

Definition at line 520 of file G4MPIscorerMerger.cc.

                                                                            {
  const G4int numEls = map->GetSize();
  G4int size = sizeof(unsigned int);
  size += sizeof(G4int)*numEls;
  //G4StatDouble: 5 doubles and 1 int
  //Can I use sizeof(G4StatDouble)? NO sizeof(G4StatDouble)==56
  size += numEls*(sizeof(G4double)*5+sizeof(G4int));
  DMSG(3,"HitStatDoubleMap "<<map<<" size: "<<size<<" in "<<numEls<<" elements.");
  return size;
}

Send()

void G4MPIscorerMerger::Send ( const unsigned int  destination )

protected

Definition at line 233 of file G4MPIscorerMerger.cc.

                                                           {
  DMSG(1,"Sending scorers "<<this);
  //Step 1: Setup buffer to pack/unpack data
  const G4int newbuffsize = CalculatePackSize(scoringManager);
  //DestroyBuffer();
  char* buffer = outputBuffer;
  if ( newbuffsize > outputBufferSize ) {
      delete[] outputBuffer;
      buffer = new char[newbuffsize];
      //Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
      //small cpu penalty, we can live with that).)
      std::fill( buffer , buffer+newbuffsize,0);
      ownsBuffer = true;
  }
  SetupOutputBuffer(buffer,newbuffsize,0);
  DMSG(3,"Buffer Size: "<<newbuffsize<< " bytes at: "<<(void*)outputBuffer);
  Pack(scoringManager);
  assert(outputBufferSize==outputBufferPosition);
 
  //Version 1: p2p communication
  comm.Send(outputBuffer, outputBufferSize, MPI::PACKED, destination, G4MPImanager::kTAG_CMDSCR);
  bytesSent += newbuffsize;
  //Receiver should use probe to get size of the package being sent
  DMSG(1,"Sending done");
}

Receive()

void G4MPIscorerMerger::Receive ( const unsigned int  source )

protected

Definition at line 205 of file G4MPIscorerMerger.cc.

                                                         {
  DMSG(1,"Receiving scorers");
 // DestroyBuffer();
      DMSG(2,"Receiving from: "<<source);
      MPI::Status status;
      comm.Probe(source, G4MPImanager::kTAG_CMDSCR, status);
      const G4int newbuffsize = status.Get_count(MPI::PACKED);
      DMSG(2,"Preparing to receive buffer of size: "<<newbuffsize);
      char* buffer = outputBuffer;
      if ( newbuffsize > outputBufferSize ) {
          DMSG(3,"New larger buffer expected, resize");
          //New larger buffer incoming, recreate buffer
          //TODO: use realloc?
          delete[] outputBuffer;
          buffer = new char[newbuffsize];
          //Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
          //small cpu penalty, we can live with that).)
          std::fill( buffer , buffer + newbuffsize , 0 );
          ownsBuffer = true;
      }
      SetupOutputBuffer(buffer,newbuffsize,0);
      comm.Recv(buffer, newbuffsize, MPI::PACKED, source,
          G4MPImanager::kTAG_CMDSCR, status);
      DMSG(3,"Buffer Size: "<<outputBufferSize<< " bytes at: "<<(void*)outputBuffer);
      UnPackAndMerge(scoringManager);
  DMSG(1,"Receiving of comamnd line scorers done");
}

Member Data Documentation

outputBuffer

char* G4MPIscorerMerger::outputBuffer

private

Definition at line 95 of file G4MPIscorerMerger.hh.

outputBufferSize

G4int G4MPIscorerMerger::outputBufferSize

private

Definition at line 96 of file G4MPIscorerMerger.hh.

outputBufferPosition

G4int G4MPIscorerMerger::outputBufferPosition

private

Definition at line 97 of file G4MPIscorerMerger.hh.

bytesSent

long G4MPIscorerMerger::bytesSent

private

Definition at line 98 of file G4MPIscorerMerger.hh.

ownsBuffer

G4bool G4MPIscorerMerger::ownsBuffer

private

Definition at line 99 of file G4MPIscorerMerger.hh.

scoringManager

G4ScoringManager* G4MPIscorerMerger::scoringManager

private

Definition at line 100 of file G4MPIscorerMerger.hh.

commSize

unsigned int G4MPIscorerMerger::commSize

private

Definition at line 101 of file G4MPIscorerMerger.hh.

destinationRank

unsigned int G4MPIscorerMerger::destinationRank

private

Definition at line 102 of file G4MPIscorerMerger.hh.

comm

MPI::Intracomm G4MPIscorerMerger::comm

private

Definition at line 103 of file G4MPIscorerMerger.hh.

verbose

G4int G4MPIscorerMerger::verbose

private

Definition at line 104 of file G4MPIscorerMerger.hh.

---

The documentation for this class was generated from the following files:

• .doxygen/Doxymodules_parallel.h
• extended/parallel/MPI/source/include/G4MPIscorerMerger.hh
• extended/parallel/MPI/source/src/G4MPIscorerMerger.cc