G4VUserMPIrunMerger Class Reference

#include <Doxymodules_parallel.h>

Inheritance diagram for G4VUserMPIrunMerger:

Classes
struct	const_registered_data
struct	registered_data

Public Member Functions
	G4VUserMPIrunMerger ()
	G4VUserMPIrunMerger (const G4Run *aRun, G4int destination=G4MPImanager::kRANK_MASTER, G4int verbosity=0)
virtual	~G4VUserMPIrunMerger ()
void	SetRun (G4Run *r)
void	SetDestinationRank (G4int i)
void	SetVerbosity (G4int ver)
virtual void	Merge ()

Protected Member Functions
virtual void	Pack ()=0
virtual G4Run *	UnPack ()=0
void	InputUserData (void *input_data, const MPI::Datatype &dt, int count)
void	OutputUserData (void *input_data, const MPI::Datatype &dt, int count)
void	SetupOutputBuffer (char *buff, G4int size, G4int position)
void	DestroyBuffer ()
G4int	GetPosition () const
char *	GetBuffer () const
G4int	GetBufferSize () const
void	Send (const unsigned int destination)
void	Receive (const unsigned int source)

Private Attributes
char *	outputBuffer
G4int	outputBufferSize
G4int	outputBufferPosition
G4bool	ownsBuffer
unsigned int	destinationRank
G4Run *	run
unsigned int	commSize
MPI::Intracomm	COMM_G4COMMAND_
G4int	verbose
long	bytesSent
std::vector< const_registered_data >	input_userdata
std::vector< registered_data >	output_userdata

Detailed Description

Definition at line 74 of file Doxymodules_parallel.h.

Constructor & Destructor Documentation

G4VUserMPIrunMerger() [1/2]

G4VUserMPIrunMerger::G4VUserMPIrunMerger

(

)

G4VUserMPIrunMerger() [2/2]

G4VUserMPIrunMerger::G4VUserMPIrunMerger	(	const G4Run *	aRun,
		G4int	destination = G4MPImanager::kRANK_MASTER,
		G4int	verbosity = 0
	)

Definition at line 34 of file G4VUserMPIrunMerger.cc.

                                           : 
    outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),
    ownsBuffer(false),
   destinationRank(destination),
   run(const_cast<G4Run*>(aRun)),
   commSize(0),
   verbose(ver),
   bytesSent(0) {}

~G4VUserMPIrunMerger()

virtual G4VUserMPIrunMerger::~G4VUserMPIrunMerger ( )

inlinevirtual

Definition at line 38 of file G4VUserMPIrunMerger.hh.

{ if ( ownsBuffer) DestroyBuffer(); }

Member Function Documentation

SetRun()

void G4VUserMPIrunMerger::SetRun ( G4Run *  r )

inline

Definition at line 39 of file G4VUserMPIrunMerger.hh.

{ run = r; }

SetDestinationRank()

void G4VUserMPIrunMerger::SetDestinationRank ( G4int  i )

inline

Definition at line 40 of file G4VUserMPIrunMerger.hh.

{ destinationRank = i; }

SetVerbosity()

void G4VUserMPIrunMerger::SetVerbosity ( G4int  ver )

inline

Definition at line 41 of file G4VUserMPIrunMerger.hh.

{ verbose = ver; }

Merge()

void G4VUserMPIrunMerger::Merge ( )

virtual

Definition at line 139 of file G4VUserMPIrunMerger.cc.

{
  // G4cout << "G4VUserMPIrunMerger::Merge called" << G4endl;
 
  DMSG(0, "G4VUserMPIrunMerger::Merge called");
  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
  const unsigned int myrank = parentComm->Get_rank();
  commSize = G4MPImanager::GetManager()->GetActiveSize();
      // do not include extra worker in this communication
 
  if ( commSize == 1 ) {
      DMSG(1,"Comm world size is 1, nothing to do");
      return;
  }
  COMM_G4COMMAND_ = parentComm->Dup();
  bytesSent = 0;
  const G4double sttime = MPI::Wtime();
 
  //Use G4MPIutils to optimize communications between ranks
  typedef std::function<void(unsigned int)> handler_t;
  using std::placeholders::_1;
  handler_t sender = std::bind(&G4VUserMPIrunMerger::Send , this , _1);
  handler_t receiver = std::bind(&G4VUserMPIrunMerger::Receive, this, _1);
  std::function<void(void)> barrier =
      std::bind(&MPI::Intracomm::Barrier,&COMM_G4COMMAND_);
  // G4cout << "go to  G4mpi::Merge" << G4endl;
  G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
 
  //OLD Style p2p communications
/*
  if ( myrank != destinationRank ) {
      DMSG(0,"Comm world size: "<<commSize<<" this rank is: "
           <<myrank<<" sending to rank "<<destinationRank);
      Send(destinationRank);
  } else {
      DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
          <<myrank<<" receiving. ");
      for ( unsigned int i = 0 ; i<commSize ; ++i) {
          if ( i != myrank ) Receive(i);
      }
  }
*/
  const G4double elapsed = MPI::Wtime() - sttime;
  long total=0;
  COMM_G4COMMAND_.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,
                                                    destinationRank);
  if ( verbose > 0 && myrank == destinationRank ) {
      //Collect from ranks how much data was sent around
      G4cout<<"G4VUserMPIrunMerger::Merge() - data transfer performances: "
                  <<double(total)/1000./elapsed<<" kB/s"
                  <<" (Total Data Transfer= "<<double(total)/1000<<" kB in "
                  <<elapsed<<" s)."<<G4endl;
  }
 
  COMM_G4COMMAND_.Free();
  DMSG(0,"G4VUserMPIrunMerger::Merge done");
}

Pack()

virtual void G4VUserMPIrunMerger::Pack ( )

protectedpure virtual

Implemented in G4MPIrunMerger.

UnPack()

virtual G4Run * G4VUserMPIrunMerger::UnPack ( )

protectedpure virtual

Implemented in G4MPIrunMerger.

InputUserData()

void G4VUserMPIrunMerger::InputUserData ( void *  input_data, const MPI::Datatype &  dt, int  count  )

inlineprotected

Definition at line 49 of file G4VUserMPIrunMerger.hh.

                                                                             {
    input_userdata.push_back( const_registered_data{input_data,dt,count} );
  }

OutputUserData()

void G4VUserMPIrunMerger::OutputUserData ( void *  input_data, const MPI::Datatype &  dt, int  count  )

inlineprotected

Definition at line 52 of file G4VUserMPIrunMerger.hh.

                                                                            {
    output_userdata.push_back( registered_data{input_data,dt,count} );
  }

SetupOutputBuffer()

void G4VUserMPIrunMerger::SetupOutputBuffer ( char *  buff, G4int  size, G4int  position  )

inlineprotected

Definition at line 58 of file G4VUserMPIrunMerger.hh.

                                                                 {
    outputBuffer = buff;
    outputBufferSize=size;
    outputBufferPosition=position;
  }

DestroyBuffer()

void G4VUserMPIrunMerger::DestroyBuffer ( )

inlineprotected

Definition at line 63 of file G4VUserMPIrunMerger.hh.

                       {
    delete[] outputBuffer;
    outputBuffer = nullptr;
    outputBufferSize=0;
    outputBufferPosition=0;
    ownsBuffer = false;
  }

GetPosition()

G4int G4VUserMPIrunMerger::GetPosition ( ) const

inlineprotected

Definition at line 71 of file G4VUserMPIrunMerger.hh.

{ return outputBufferPosition; }

GetBuffer()

char * G4VUserMPIrunMerger::GetBuffer ( ) const

inlineprotected

Definition at line 72 of file G4VUserMPIrunMerger.hh.

{ return outputBuffer; }

GetBufferSize()

G4int G4VUserMPIrunMerger::GetBufferSize ( ) const

inlineprotected

Definition at line 73 of file G4VUserMPIrunMerger.hh.

{ return outputBufferSize; }

Send()

void G4VUserMPIrunMerger::Send ( const unsigned int  destination )

protected

Definition at line 47 of file G4VUserMPIrunMerger.cc.

{
  assert(run!=nullptr);
  G4int nevts = run->GetNumberOfEvent();
  DMSG( 1 , "G4VUserMPIrunMerger::Send() : Sending a G4run ("
        <<run<<") with "<<nevts<<" events to: "<<destination);
  input_userdata.clear();
  Pack();//User code
  InputUserData(&nevts,MPI::INT,1);
 
  DestroyBuffer();
  G4int newbuffsize = 0;
  for ( const const_registered_data& el : input_userdata ) {
      newbuffsize += (el.dt.Get_size()*el.count);
  }
  char* buffer = new char[newbuffsize];
  //Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
  //small cpu penalty, we can live with that).)
  std::fill(buffer,buffer+newbuffsize,0);
  ownsBuffer=true;
  SetupOutputBuffer(buffer,newbuffsize,0);
  DMSG(3,"Buffer size: "<<newbuffsize<<" bytes at: "<<(void*)outputBuffer);
 
  //Now userdata contains all data to be send, do the real packing
  for ( const const_registered_data& el : input_userdata ) {
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
      MPI_Pack(const_cast<void*>(el.p_data),el.count,el.dt,
#else
      MPI_Pack(el.p_data,el.count,el.dt,
#endif
          outputBuffer,outputBufferSize,
          &outputBufferPosition,COMM_G4COMMAND_);
  }
  assert(outputBufferSize==outputBufferPosition);
  COMM_G4COMMAND_.Send(outputBuffer , outputBufferSize , MPI::PACKED ,
                                                destination , G4MPImanager::kTAG_RUN);
  bytesSent+=outputBufferSize;
  DMSG(2 , "G4VUserMPIrunMerger::Send() : Done ");
}

Receive()

void G4VUserMPIrunMerger::Receive ( const unsigned int  source )

protected

Definition at line 88 of file G4VUserMPIrunMerger.cc.

{
  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
  DMSG( 1 , "G4VUserMPIrunMerger::Receive(...) , this rank : "
            <<parentComm->Get_rank()<<" and receiving from : "<<source);
  //DestroyBuffer();
  //Receive from all but one
  //for (G4int rank = 0; rank < commSize-1; ++rank)
    //{
      MPI::Status status;
      COMM_G4COMMAND_.Probe(source, G4MPImanager::kTAG_RUN, status);
      //const G4int source = status.Get_source();
      const G4int newbuffsize = status.Get_count(MPI::PACKED);
      DMSG(2,"Preparing to receive buffer of size: "<<newbuffsize);
      char* buffer = outputBuffer;
      if ( newbuffsize > outputBufferSize ) {
          DMSG(3,"New larger buffer expected, resize");
          //New larger buffer incoming, recreate buffer
          delete[] outputBuffer;
          buffer = new char[newbuffsize];
          //Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
          //small cpu penalty, we can live with that).)
          std::fill(buffer,buffer+newbuffsize,0);
          ownsBuffer = true;
      }
      SetupOutputBuffer(buffer,newbuffsize,0);
      COMM_G4COMMAND_.Recv(buffer, newbuffsize, MPI::PACKED,source,
          G4MPImanager::kTAG_RUN, status);
      DMSG(3,"Buffer Size: "<<outputBufferSize<< " bytes at: "<<(void*)outputBuffer);
      output_userdata.clear();
      //User code, if implemented will return the concrete G4Run class
      G4Run* aNewRun = UnPack();
      if ( aNewRun == nullptr ) aNewRun = new G4Run;
      //Add number of events counter
      G4int nevets = 0;
      OutputUserData(&nevets,MPI::INT,1);
      //now userdata contains all data references, do the real unpacking
      for ( const registered_data& el : output_userdata ) {
          MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
              el.p_data,el.count,el.dt,COMM_G4COMMAND_);
      }
      for ( G4int i = 0 ; i<nevets ; ++i ) aNewRun->RecordEvent(nullptr);
 
      //Now merge received MPI run with global one
      DMSG(2,"Before G4Run::Merge : "<<run->GetNumberOfEvent());
      run->Merge( aNewRun );
      DMSG(2,"After G4Run::Merge : "<<run->GetNumberOfEvent());
      delete aNewRun;
    //}
}

Member Data Documentation

outputBuffer

char* G4VUserMPIrunMerger::outputBuffer

private

Definition at line 78 of file G4VUserMPIrunMerger.hh.

outputBufferSize

G4int G4VUserMPIrunMerger::outputBufferSize

private

Definition at line 79 of file G4VUserMPIrunMerger.hh.

outputBufferPosition

G4int G4VUserMPIrunMerger::outputBufferPosition

private

Definition at line 80 of file G4VUserMPIrunMerger.hh.

ownsBuffer

G4bool G4VUserMPIrunMerger::ownsBuffer

private

Definition at line 81 of file G4VUserMPIrunMerger.hh.

destinationRank

unsigned int G4VUserMPIrunMerger::destinationRank

private

Definition at line 82 of file G4VUserMPIrunMerger.hh.

run

G4Run* G4VUserMPIrunMerger::run

private

Definition at line 83 of file G4VUserMPIrunMerger.hh.

commSize

unsigned int G4VUserMPIrunMerger::commSize

private

Definition at line 84 of file G4VUserMPIrunMerger.hh.

COMM_G4COMMAND_

MPI::Intracomm G4VUserMPIrunMerger::COMM_G4COMMAND_

private

Definition at line 85 of file G4VUserMPIrunMerger.hh.

verbose

G4int G4VUserMPIrunMerger::verbose

private

Definition at line 86 of file G4VUserMPIrunMerger.hh.

bytesSent

long G4VUserMPIrunMerger::bytesSent

private

Definition at line 87 of file G4VUserMPIrunMerger.hh.

input_userdata

std::vector<const_registered_data> G4VUserMPIrunMerger::input_userdata

private

Definition at line 99 of file G4VUserMPIrunMerger.hh.

output_userdata

std::vector<registered_data> G4VUserMPIrunMerger::output_userdata

private

Definition at line 109 of file G4VUserMPIrunMerger.hh.

---

The documentation for this class was generated from the following files:

• .doxygen/Doxymodules_parallel.h
• extended/parallel/MPI/source/include/G4VUserMPIrunMerger.hh
• extended/parallel/MPI/source/src/G4VUserMPIrunMerger.cc