G4MPImanager Class Reference

#include <Doxymodules_parallel.h>

Public Types
enum	{ kRANK_MASTER = 0 }
enum	{   kTAG_G4COMMAND = 100 , kTAG_G4STATUS = 200 , kTAG_G4SEED = 300 , kTAG_DATA = 1000 ,   kTAG_HISTO = 1001 , kTAG_RUN = 1002 , kTAG_CMDSCR = 1003 , kTAG_NTUPLE = 1004 }

Public Member Functions
	G4MPImanager (int nof_extra_workers=0)
	G4MPImanager (int argc, char **argv, int nof_extra_workers=0)
	~G4MPImanager ()
G4MPIsession *	GetMPIsession () const
G4int	GetVerbose () const
void	SetVerbose (G4int iverbose)
G4int	GetTotalSize () const
G4int	GetActiveSize () const
G4int	GetRank () const
G4bool	IsMaster () const
G4bool	IsSlave () const
G4bool	IsExtraWorker () const
G4bool	IsInitMacro () const
const G4String &	GetInitFileName () const
G4bool	IsBatchMode () const
const G4String &	GetMacroFileName () const
void	SetMasterWeight (G4double aweight)
G4double	GetMasterWeight () const
void	SetExtraWorker (G4VMPIextraWorker *extraWorker)
G4VMPIextraWorker *	GetExtraWorker () const
G4VMPIseedGenerator *	GetSeedGenerator () const
G4String	BcastCommand (const G4String &command)
void	ShowStatus ()
void	ShowSeeds ()
void	SetSeed (G4int inode, G4long seed)
void	WaitBeamOn ()
void	DistributeSeeds ()
void	ExecuteMacroFile (const G4String &fname, G4bool qbatch=false)
G4bool	CheckThreadStatus ()
void	ExecuteThreadCommand (const G4String &command)
void	ExecuteBeamOnThread (const G4String &command)
void	JoinBeamOnThread ()
void	BeamOn (G4int nevent, G4bool qdivide=true)
void	Print (const G4String &message)
G4int	GetEventsInMaster () const
G4int	GetEventsInSlave () const
void	ShowHelp () const
const MPI::Intracomm *	GetComm () const
const MPI_Comm *	GetProcessingComm () const
const MPI_Comm *	GetCollectingComm () const
const MPI_Comm *	GetAllComm () const

Static Public Member Functions
static G4MPImanager *	GetManager ()

Private Member Functions
	DISALLOW_COPY_AND_ASSIGN (G4MPImanager)
void	Initialize ()
void	ParseArguments (G4int argc, char **argv)
void	UpdateStatus ()

Private Attributes
G4MPImessenger *	messenger_
G4MPIsession *	session_
G4VMPIextraWorker *	extra_worker_
G4VMPIseedGenerator *	seed_generator_
G4MPIstatus *	status_
G4int	verbose_
G4bool	is_master_
G4bool	is_slave_
G4bool	is_extra_worker_
G4int	rank_
G4int	size_
G4int	world_size_
MPI::Intracomm	COMM_G4COMMAND_
MPI_Comm	processing_comm_
MPI_Comm	collecting_comm_
MPI_Comm	all_comm_
MPI_Group	world_group_
MPI_Group	processing_group_
MPI_Group	collecting_group_
MPI_Group	all_group_
G4bool	qfcout_
std::ofstream	fscout_
G4bool	qinitmacro_
G4String	init_file_name_
G4bool	qbatchmode_
G4String	macro_file_name_
pthread_t	thread_id_
G4int	fevents_in_master = 0
G4int	fevents_in_slave = 0
G4double	master_weight_
G4int	nof_extra_workers_

Static Private Attributes
static G4MPImanager *	g4mpi_ = NULL

Detailed Description

Definition at line 63 of file Doxymodules_parallel.h.

Member Enumeration Documentation

anonymous enum

anonymous enum

Enumerator
kRANK_MASTER

Definition at line 49 of file G4MPImanager.hh.

{ kRANK_MASTER = 0 };

anonymous enum

anonymous enum

Enumerator
kTAG_G4COMMAND
kTAG_G4STATUS
kTAG_G4SEED
kTAG_DATA
kTAG_HISTO
kTAG_RUN
kTAG_CMDSCR
kTAG_NTUPLE

Definition at line 51 of file G4MPImanager.hh.

       { // MPI tag
    kTAG_G4COMMAND = 100,
    kTAG_G4STATUS = 200,
    kTAG_G4SEED = 300,
    kTAG_DATA = 1000,
    kTAG_HISTO = 1001,
    kTAG_RUN = 1002,
    kTAG_CMDSCR = 1003,
    kTAG_NTUPLE = 1004
  };

Constructor & Destructor Documentation

G4MPImanager() [1/2]

G4MPImanager::G4MPImanager

(

int

nof_extra_workers = 0

)

Definition at line 69 of file G4MPImanager.cc.

  : verbose_(0), 
    COMM_G4COMMAND_(MPI_COMM_NULL), processing_comm_(MPI_COMM_NULL), 
    collecting_comm_(MPI_COMM_NULL), all_comm_(MPI_COMM_NULL),
    qfcout_(false), qinitmacro_(false), qbatchmode_(false),
    thread_id_(0), master_weight_(1.), nof_extra_workers_(nof_extra_workers)
{
  //MPI::Init();
  MPI::Init_thread(MPI::THREAD_SERIALIZED);
  Initialize();
}

G4MPImanager() [2/2]

G4MPImanager::G4MPImanager	(	int	argc,
		char **	argv,
		int	nof_extra_workers = 0
	)

Definition at line 82 of file G4MPImanager.cc.

  : verbose_(0), 
    COMM_G4COMMAND_(MPI_COMM_NULL), processing_comm_(MPI_COMM_NULL), 
    collecting_comm_(MPI_COMM_NULL), all_comm_(MPI_COMM_NULL),
    qfcout_(false), qinitmacro_(false), qbatchmode_(false),
    thread_id_(0), master_weight_(1.), nof_extra_workers_(nof_extra_workers)
{
  //MPI::Init(argc, argv);
  MPI::Init_thread(argc, argv, MPI::THREAD_SERIALIZED);
  Initialize();
  ParseArguments(argc, argv);
}

~G4MPImanager()

G4MPImanager::~G4MPImanager

(

)

Definition at line 96 of file G4MPImanager.cc.

{
  if( is_slave_ && qfcout_ ) fscout_.close();
 
  delete status_;
  delete messenger_;
  delete session_;
 
  if ( nof_extra_workers_ ) {
    MPI_Group_free(&world_group_);
    MPI_Group_free(&processing_group_);
    MPI_Group_free(&collecting_group_);
    MPI_Group_free(&all_group_);
    if (processing_comm_ != MPI_COMM_NULL) {
      MPI_Comm_free(&processing_comm_);
    }
    if (collecting_comm_ != MPI_COMM_NULL) {
      MPI_Comm_free(&collecting_comm_);
    }
    if (all_comm_ != MPI_COMM_NULL) {
      MPI_Comm_free(&all_comm_);
    }
  }
  else {
    COMM_G4COMMAND_.Free();
  }
 
  MPI::Finalize();
}

Member Function Documentation

GetManager()

G4MPImanager * G4MPImanager::GetManager ( )

static

Definition at line 127 of file G4MPImanager.cc.

{
  if ( g4mpi_ == NULL ) {
    G4Exception("G4MPImanager::GetManager()", "MPI001",
                FatalException, "G4MPImanager is not instantiated.");
  }
  return g4mpi_;
}

GetMPIsession()

G4MPIsession * G4MPImanager::GetMPIsession ( ) const

inline

Definition at line 186 of file G4MPImanager.hh.

{
  return session_;
}

GetVerbose()

G4int G4MPImanager::GetVerbose ( ) const

inline

Definition at line 191 of file G4MPImanager.hh.

{
  return verbose_;
}

SetVerbose()

void G4MPImanager::SetVerbose ( G4int  iverbose )

inline

Definition at line 196 of file G4MPImanager.hh.

{
  G4int lv = iverbose;
  if( iverbose > 1 ) lv = 1;
  if( iverbose < 0 ) lv = 0;
 
  verbose_ = lv;
  return;
}

GetTotalSize()

G4int G4MPImanager::GetTotalSize ( ) const

inline

Definition at line 211 of file G4MPImanager.hh.

{
  return world_size_;
}

GetActiveSize()

G4int G4MPImanager::GetActiveSize ( ) const

inline

Definition at line 216 of file G4MPImanager.hh.

{
  return size_;
}

GetRank()

G4int G4MPImanager::GetRank ( ) const

inline

Definition at line 206 of file G4MPImanager.hh.

{
  return rank_;
}

IsMaster()

G4bool G4MPImanager::IsMaster ( ) const

inline

Definition at line 221 of file G4MPImanager.hh.

{
  return is_master_;
}

IsSlave()

G4bool G4MPImanager::IsSlave ( ) const

inline

Definition at line 226 of file G4MPImanager.hh.

{
  return is_slave_;
}

IsExtraWorker()

G4bool G4MPImanager::IsExtraWorker ( ) const

inline

Definition at line 231 of file G4MPImanager.hh.

{
  return is_extra_worker_;
}

IsInitMacro()

G4bool G4MPImanager::IsInitMacro ( ) const

inline

Definition at line 236 of file G4MPImanager.hh.

{
  return qinitmacro_;
 
}

GetInitFileName()

const G4String & G4MPImanager::GetInitFileName ( ) const

inline

Definition at line 242 of file G4MPImanager.hh.

{
  return init_file_name_;
 
}

IsBatchMode()

G4bool G4MPImanager::IsBatchMode ( ) const

inline

Definition at line 248 of file G4MPImanager.hh.

{
  return qbatchmode_;
}

GetMacroFileName()

const G4String & G4MPImanager::GetMacroFileName ( ) const

inline

Definition at line 253 of file G4MPImanager.hh.

{
  return macro_file_name_;
}

SetMasterWeight()

void G4MPImanager::SetMasterWeight ( G4double  aweight )

inline

Definition at line 258 of file G4MPImanager.hh.

{
  master_weight_ = aweight;
 
  if( aweight < 0. ) master_weight_ = 0.;
  if( aweight > 1. ) master_weight_ = 1.;
}

GetMasterWeight()

G4double G4MPImanager::GetMasterWeight ( ) const

inline

Definition at line 266 of file G4MPImanager.hh.

{
  return master_weight_;
}

SetExtraWorker()

void G4MPImanager::SetExtraWorker

(

G4VMPIextraWorker *

extraWorker

)

Definition at line 137 of file G4MPImanager.cc.

{
  if ( ! nof_extra_workers_ ) {
    G4Exception("G4MPImanager::SetExtraWorker()", "MPI001",
      FatalException, "Number of extra workers >0 must be set first.");
  }
 
  extra_worker_ = extraWorker;
}

GetExtraWorker()

G4VMPIextraWorker * G4MPImanager::GetExtraWorker ( ) const

inline

Definition at line 271 of file G4MPImanager.hh.

{
  return extra_worker_;
}

GetSeedGenerator()

G4VMPIseedGenerator * G4MPImanager::GetSeedGenerator ( ) const

inline

Definition at line 276 of file G4MPImanager.hh.

{
  return seed_generator_;
}

BcastCommand()

G4String G4MPImanager::BcastCommand

(

const G4String &

command

)

Definition at line 515 of file G4MPImanager.cc.

{
  // Do nothing if not processing worker
  if (is_extra_worker_) return G4String("exit");
 
  enum { kBUFF_SIZE = 512 };
  static char sbuff[kBUFF_SIZE];
  command.copy(sbuff, kBUFF_SIZE);
  G4int len = command.size();
  sbuff[len] ='\0';   // no boundary check
 
  // "command" is not yet fixed in slaves at this time.
 
  // waiting message exhausts CPU in LAM!
  //COMM_G4COMMAND_.Bcast(sbuff, ssize, MPI::CHAR, RANK_MASTER);
 
  // another implementation
  if( is_master_ ) {
    for ( G4int islave = 1; islave < size_; islave++ ) {
      COMM_G4COMMAND_.Send(sbuff, kBUFF_SIZE, MPI::CHAR,
                           islave, kTAG_G4COMMAND);
    }
  } else {
    // try non-blocking receive
    MPI::Request request= COMM_G4COMMAND_.Irecv(sbuff, kBUFF_SIZE, MPI::CHAR,
                                                kRANK_MASTER, kTAG_G4COMMAND);
    // polling...
    while(! request.Test()) {
      ::Wait(1000);
    }
  }
 
  return G4String(sbuff);
}

ShowStatus()

void G4MPImanager::ShowStatus

(

)

Definition at line 334 of file G4MPImanager.cc.

{
  G4int buff[G4MPIstatus::kNSIZE];
 
  UpdateStatus();
  G4bool gstatus = CheckThreadStatus();
 
  if ( is_master_ ) {
    status_-> Print();  // for maser itself
 
    G4int nev = status_-> GetEventID();
    G4int nevtp = status_-> GetNEventToBeProcessed();
    G4double cputime = status_-> GetCPUTime();
 
    // receive from each slave
    for ( G4int islave = 1; islave < size_; islave++ ) {
      COMM_G4COMMAND_.Recv(buff, G4MPIstatus::kNSIZE, MPI::INT,
                          islave, kTAG_G4STATUS);
      status_-> UnPack(buff);
      status_-> Print();
 
      // aggregation
      nev += status_-> GetEventID();
      nevtp += status_-> GetNEventToBeProcessed();
      cputime += status_-> GetCPUTime();
    }
 
    G4String strStatus;
    if ( gstatus ) {
      strStatus = "Run";
    } else {
      strStatus = "Idle";
    }
 
    G4cout << "-------------------------------------------------------"
           << G4endl
           << "* #ranks= " << size_
           << "   event= " << nev << "/" << nevtp
           << " state= " << strStatus
           << " time= " << cputime << "s"
           << G4endl;
  } else {
    status_-> Pack(buff);
    COMM_G4COMMAND_.Send(buff, G4MPIstatus::kNSIZE, MPI::INT,
                         kRANK_MASTER, kTAG_G4STATUS);
  }
}

ShowSeeds()

void G4MPImanager::ShowSeeds

(

)

Definition at line 393 of file G4MPImanager.cc.

{
  G4long buff;
 
  if ( is_master_ ) {
    // print master
    G4cout << "* rank= " << rank_
           << " seed= " << G4Random::getTheSeed()
           << G4endl;
    // receive from each slave
    for ( G4int islave = 1; islave < size_; islave++ ) {
      COMM_G4COMMAND_.Recv(&buff, 1, MPI::LONG, islave, kTAG_G4SEED);
      G4cout << "* rank= " << islave
             << " seed= " << buff
             << G4endl;
    }
  } else { // slaves
    buff = G4Random::getTheSeed();
    COMM_G4COMMAND_.Send(&buff, 1, MPI::LONG, kRANK_MASTER, kTAG_G4SEED);
  }
}

SetSeed()

void G4MPImanager::SetSeed	(	G4int	inode,
		G4long	seed
	)

Definition at line 416 of file G4MPImanager.cc.

{
  if( rank_ == inode ) {
    CLHEP::HepRandom::setTheSeed(seed);
  }
}

WaitBeamOn()

void G4MPImanager::WaitBeamOn

(

)

Definition at line 617 of file G4MPImanager.cc.

{
  // G4cout << "G4MPImanager::WaitBeamOn" << G4endl;
 
  // Extra worker
  if (is_extra_worker_) {
    if ( extra_worker_ ) {
      G4cout << "Calling extra_worker " << G4endl;
      extra_worker_->BeamOn();
    } else {
       G4cout << " !!!! extra_worker_ is not defined " << G4endl;
    }
    return;
  }
 
  G4int buff = 0;
  if ( qbatchmode_ ) {  // valid only in batch mode
    if ( is_master_ ) {
      // receive from each slave
      for (G4int islave = 1; islave < size_; islave++) {
        // G4cout << "calling Irecv for islave " << islave << G4endl; 
        MPI::Request request = COMM_G4COMMAND_.Irecv(&buff, 1, MPI::INT,
                                                     islave, kTAG_G4STATUS);
        while(! request.Test()) {
           ::Wait(1000);
        }
      }
    } else {
      buff = 1;
      // G4cout << "calling send for i " << kRANK_MASTER << G4endl; 
      COMM_G4COMMAND_.Send(&buff, 1, MPI::INT, kRANK_MASTER, kTAG_G4STATUS);
    }
  }
}

DistributeSeeds()

void G4MPImanager::DistributeSeeds

(

)

Definition at line 383 of file G4MPImanager.cc.

{
  // Do nothing if not processing worker
  if ( is_extra_worker_ ) return;
 
  std::vector<G4long> seed_list = seed_generator_-> GetSeedList();
  G4Random::setTheSeed(seed_list[rank_]);
}

ExecuteMacroFile()

void G4MPImanager::ExecuteMacroFile	(	const G4String &	fname,
		G4bool	qbatch = false
	)

Definition at line 551 of file G4MPImanager.cc.

{
  G4bool currentmode = qbatchmode_;
  qbatchmode_ = true;
  G4MPIbatch* batchSession = new G4MPIbatch(fname, qbatch);
  batchSession-> SessionStart();
  delete batchSession;
  qbatchmode_ = currentmode;
}

CheckThreadStatus()

G4bool G4MPImanager::CheckThreadStatus

(

)

Definition at line 424 of file G4MPImanager.cc.

{
  unsigned buff;
  unsigned qstatus = 0;
 
  if( is_master_ ) {
    qstatus = (thread_id_ != 0);
    // get slave status
    for ( G4int islave = 1; islave < size_; islave++ ) {
      MPI::Request request = COMM_G4COMMAND_.Irecv(&buff, 1, MPI::UNSIGNED,
                                                  islave, kTAG_G4STATUS);
      while( ! request.Test() ) {
        ::Wait(1000);
      }
      qstatus |= buff;
    }
  } else {
    buff = (thread_id_ !=0);
    COMM_G4COMMAND_.Send(&buff, 1, MPI::UNSIGNED, kRANK_MASTER, kTAG_G4STATUS);
  }
 
  // broadcast
  buff = qstatus; // for master
  COMM_G4COMMAND_.Bcast(&buff, 1, MPI::UNSIGNED, kRANK_MASTER);
  qstatus = buff; // for slave
 
  if ( qstatus != 0 ) return true;
  else return false;
}

ExecuteThreadCommand()

void G4MPImanager::ExecuteThreadCommand

(

const G4String &

command

)

Definition at line 455 of file G4MPImanager.cc.

{
  // this method is a thread function.
  G4UImanager* UI = G4UImanager::GetUIpointer();
  G4int rc = UI-> ApplyCommand(command);
 
  G4int commandStatus = rc - (rc%100);
 
  switch( commandStatus ) {
  case fCommandSucceeded:
    break;
  case fIllegalApplicationState:
    G4cerr << "illegal application state -- command refused" << G4endl;
    break;
  default:
    G4cerr << "command refused (" << commandStatus << ")" << G4endl;
    break;
  }
 
  // thread is joined
  if ( thread_id_ ) {
    pthread_join(thread_id_, 0);
    thread_id_ = 0;
  }
 
  return;
}

ExecuteBeamOnThread()

void G4MPImanager::ExecuteBeamOnThread

(

const G4String &

command

)

Definition at line 484 of file G4MPImanager.cc.

{
  G4bool threadStatus = CheckThreadStatus();
 
  if (threadStatus) {
    if ( is_master_ ) {
      G4cout << "G4MPIsession:: beamOn is still running." << G4endl;
    }
  } else { // ok
    static G4String cmdstr;
    cmdstr = command;
    G4int rc = pthread_create(&thread_id_, 0,
                              (Func_t)thread_ExecuteThreadCommand,
                              (void*)&cmdstr);
    if (rc != 0)
      G4Exception("G4MPImanager::ExecuteBeamOnThread()",
                  "MPI003", FatalException,
                  "Failed to create a beamOn thread.");
  }
}

JoinBeamOnThread()

void G4MPImanager::JoinBeamOnThread

(

)

Definition at line 506 of file G4MPImanager.cc.

{
  if ( thread_id_ ) {
    pthread_join(thread_id_, 0);
    thread_id_ = 0;
  }
}

BeamOn()

void G4MPImanager::BeamOn	(	G4int	nevent,
		G4bool	qdivide = true
	)

Definition at line 562 of file G4MPImanager.cc.

{
  // G4cout << "G4MPImanager::BeamOn " << nevent << G4endl;
 
#ifndef G4MULTITHREADED
  G4RunManager* runManager = G4RunManager::GetRunManager();
#endif
 
  if ( qdivide ) { // events are divided
    G4double ntot = master_weight_ + size_ - 1.;
    G4int nproc = G4int(nevent/ntot);
    G4int nproc0 = nevent - nproc*(size_ - 1);
    fevents_in_master = nproc0;
    fevents_in_slave = nproc;
 
    if ( verbose_ > 0 && is_master_ ) {
      G4cout << "#events in master=" << nproc0 << " / "
             << "#events in slave="  << nproc << G4endl;
    }
 
    status_-> StartTimer(); // start timer
 
#ifdef G4MULTITHREADED
    G4String str_nevt;
    if ( is_master_ ) str_nevt = G4UIcommand::ConvertToString(nproc0);
    else str_nevt = G4UIcommand::ConvertToString(nproc);
    G4UImanager* UI = G4UImanager::GetUIpointer();
    UI-> ApplyCommand("/run/beamOn " + str_nevt);
#else
    if ( is_master_ ) runManager-> BeamOn(nproc0);
    else runManager-> BeamOn(nproc);
#endif
 
    status_-> StopTimer(); // stop timer
 
  } else { // same events are generated in each node (for test use)
    if( verbose_ > 0 && is_master_ ) {
      G4cout << "#events in master=" << nevent << " / "
             << "#events in slave="  << nevent << G4endl;
    }
    status_-> StartTimer(); // start timer
 
#ifdef G4MULTITHREADED
    G4String str_nevt = G4UIcommand::ConvertToString(nevent);
    G4UImanager* UI = G4UImanager::GetUIpointer();
    UI-> ApplyCommand("/run/beamOn " + str_nevt);
#else
    runManager-> BeamOn(nevent);
#endif
 
    status_-> StopTimer(); // stop timer
  }
}

Print()

void G4MPImanager::Print

(

const G4String &

message

)

Definition at line 653 of file G4MPImanager.cc.

{
  if ( is_master_ ){
    std::cout << message << std::flush;
  } else {
    if ( qfcout_ ) { // output to a file
      fscout_ << message << std::flush;
    } else { // output to stdout
      std::cout << rank_ << ":" << message << std::flush;
    }
  }
}

GetEventsInMaster()

G4int G4MPImanager::GetEventsInMaster ( ) const

inline

Definition at line 113 of file G4MPImanager.hh.

{return fevents_in_master;}

GetEventsInSlave()

G4int G4MPImanager::GetEventsInSlave ( ) const

inline

Definition at line 114 of file G4MPImanager.hh.

{return fevents_in_slave;}

ShowHelp()

void G4MPImanager::ShowHelp

(

)

const

Definition at line 667 of file G4MPImanager.cc.

{
  if (is_slave_ ) return;
 
  G4cout << "Geant4 MPI interface" << G4endl;
  G4cout << "usage:" << G4endl;
  G4cout << "<app> [options] [macro file]"
         << G4endl << G4endl;
  G4cout << "   -h, --help              show this message."
         << G4endl;
  G4cout << "   -v, --verbose           show verbose message"
         << G4endl;
  G4cout << "   -i, --init=FNAME        set an init macro file"
         << G4endl;
  G4cout << "   -o, --ofile[=FNAME]     set slave output to a flie"
         << G4endl;
  G4cout << G4endl;
}

GetComm()

const MPI::Intracomm * G4MPImanager::GetComm ( ) const

inline

Definition at line 119 of file G4MPImanager.hh.

{ return &COMM_G4COMMAND_; }

GetProcessingComm()

const MPI_Comm * G4MPImanager::GetProcessingComm ( ) const

inline

Definition at line 120 of file G4MPImanager.hh.

{ return &processing_comm_; }

GetCollectingComm()

const MPI_Comm * G4MPImanager::GetCollectingComm ( ) const

inline

Definition at line 121 of file G4MPImanager.hh.

{ return &collecting_comm_; }

GetAllComm()

const MPI_Comm * G4MPImanager::GetAllComm ( ) const

inline

Definition at line 122 of file G4MPImanager.hh.

{ return &all_comm_; }

DISALLOW_COPY_AND_ASSIGN()

G4MPImanager::DISALLOW_COPY_AND_ASSIGN ( G4MPImanager  )

private

Initialize()

void G4MPImanager::Initialize ( )

private

Definition at line 148 of file G4MPImanager.cc.

{
  // G4cout << "G4MPImanager::Initialize" << G4endl;
 
  if ( g4mpi_ != NULL ) {
    G4Exception("G4MPImanager::Initialize()", "MPI002",
                FatalException, "G4MPImanager is already instantiated.");
  }
 
  g4mpi_ = this;
 
  // get rank information
  world_size_ = MPI::COMM_WORLD.Get_size();
  if ( world_size_ - nof_extra_workers_ <= 0 ) {
    G4Exception("G4MPImanager::SetExtraWorker()", "MPI001",
    JustWarning, "Cannot reserve extra ranks: the MPI size is not sufficient.");
    nof_extra_workers_ = 0;
  }
  size_ = world_size_ - nof_extra_workers_;
  rank_ = MPI::COMM_WORLD.Get_rank();
  is_master_ = (rank_ == kRANK_MASTER);
  is_slave_ = (rank_ != kRANK_MASTER);
  is_extra_worker_ = false;
 
  if ( nof_extra_workers_ ) {
    // G4cout << "Extra workers requested" << G4endl;
 
    // Define three groups of workers: processing, collecting and all;
    // if no extra workers are declared, all world ranks are processing ranks
 
    // MPI_Group world_group;
    MPI_Comm_group(MPI_COMM_WORLD, &world_group_);
    
    // Group 1 - processing ranks
    int* ranks1 =  new int[size_];
    for (int i=0; i<size_; i++) ranks1[i] = i;
    // Construct a group containing all of the processing ranks in world_group
    MPI_Group_incl(world_group_, size_, ranks1, &processing_group_);
    
    // Group 2 - collecting ranks
    int* ranks2 =  new int[nof_extra_workers_];
    for (int i=0; i<nof_extra_workers_; i++) ranks2[i] = (world_size_ - nof_extra_workers_)  + i;
    // Construct a group containing all of the collecting ranks in world_group
    MPI_Group_incl(world_group_, nof_extra_workers_, ranks2, &collecting_group_);
    
    // Group 3 - all ranks
    int* ranks3 = new int[world_size_];
    for (int i=0; i<world_size_; i++) ranks3[i] = i;
    // Construct a group containing all of the processing ranks in world_group
    MPI_Group_incl(world_group_, world_size_, ranks3, &all_group_);
 
    // Create new communicators based on the groups
    MPI_Comm_create_group(MPI_COMM_WORLD, processing_group_, 0, &processing_comm_);
    MPI_Comm_create_group(MPI_COMM_WORLD, collecting_group_, 0, &collecting_comm_);
    MPI_Comm_create_group(MPI_COMM_WORLD, all_group_, 0, &all_comm_);
 
    // COMM_G4COMMAND_ = processing_comm_ copy
    COMM_G4COMMAND_ = MPI::Intracomm(processing_comm_);
 
  } else {
    // G4cout << "No extra workers requested" << G4endl;
    // initialize MPI communicator
    COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
  }
 
  is_extra_worker_ = (collecting_comm_ != MPI_COMM_NULL);
 
  // new G4MPI stuffs
  messenger_ = new G4MPImessenger();
  messenger_-> SetTargetObject(this);
  session_ = new G4MPIsession;
  status_ = new G4MPIstatus;
 
  if ( ! is_extra_worker_ ) {
    // default seed generator is random generator.
    seed_generator_ = new G4MPIrandomSeedGenerator;
    DistributeSeeds();
  }
 
  // print status of this worker
  // G4cout << this << " world_size_ " << world_size_ << G4endl;
  // G4cout << this << " size_ " << size_ << G4endl;
  // G4cout << this << " nof_extra_workers_  " << nof_extra_workers_  << G4endl;
  // G4cout << this << " is_master_ " << is_master_ << G4endl;
  // G4cout << this << " is_slave_ " << is_slave_ << G4endl;
  // G4cout << this << " is_extra_worker_ " << is_extra_worker_ << G4endl;
  // G4cout << this << " is_processing_worker_ " 
  //        <<  (processing_comm_ != MPI_COMM_NULL) << G4endl;
}

ParseArguments()

void G4MPImanager::ParseArguments ( G4int  argc, char **  argv  )

private

Definition at line 239 of file G4MPImanager.cc.

{
  G4int qhelp = 0;
  G4String ofprefix = "mpi";
 
  G4int c;
  while ( 1 ) {
    G4int option_index = 0;
    static struct option long_options[] = {
      {"help",    no_argument,        NULL, 'h'},
      {"verbose", no_argument,        NULL, 'v'},
      {"init",    required_argument,  NULL, 'i'},
      {"ofile",   optional_argument,  NULL, 'o'},
      {NULL,      0,                  NULL,  0}
    };
 
    opterr = 0; // suppress message
    c = getopt_long(argc, argv, "hvi:o", long_options, &option_index);
    opterr = 1;
 
    if( c == -1 ) break;
 
    switch (c) {
    case 'h' :
      qhelp = 1;
      break;
    case 'v' :
      verbose_ = 1;
      break;
    case 'i' :
      qinitmacro_ = true;
      init_file_name_ = optarg;
      break;
    case 'o' :
      qfcout_ = true;
      if ( optarg ) ofprefix = optarg;
      break;
    default:
      G4cerr << "*** invalid options specified." << G4endl;
      std::exit(EXIT_FAILURE);
      break;
    }
  }
 
  // show help
  if ( qhelp ) {
    if ( is_master_ ) ShowHelp();
    MPI::Finalize();
    std::exit(EXIT_SUCCESS);
  }
 
  // file output
  if( is_slave_ && qfcout_ ) {
    G4String prefix = ofprefix + ".%03d" + ".cout";
    char str[1024];
    sprintf(str, prefix.c_str(), rank_);
    G4String fname(str);
    fscout_.open(fname.c_str(), std::ios::out);
  }
 
  // non-option ARGV-elements ...
  if ( optind < argc ) {
    qbatchmode_ = true;
    macro_file_name_ = argv[optind];
  }
}

UpdateStatus()

void G4MPImanager::UpdateStatus ( )

private

Definition at line 307 of file G4MPImanager.cc.

{
  G4RunManager* runManager = G4RunManager::GetRunManager();
  const G4Run* run = runManager-> GetCurrentRun();
 
  G4int runid, eventid, neventTBP;
 
  G4StateManager* stateManager = G4StateManager::GetStateManager();
  G4ApplicationState g4state = stateManager-> GetCurrentState();
 
  if ( run ) {
    runid = run-> GetRunID();
    neventTBP = run -> GetNumberOfEventToBeProcessed();
    eventid = run-> GetNumberOfEvent();
    if( g4state == G4State_GeomClosed || g4state == G4State_EventProc ) {
      status_-> StopTimer();
    }
  } else {
    runid = 0;
    eventid = 0;
    neventTBP = 0;
  }
 
  status_-> SetStatus(rank_, runid, neventTBP, eventid, g4state);
}

Member Data Documentation

g4mpi_

G4MPImanager * G4MPImanager::g4mpi_ = NULL

staticprivate

Definition at line 131 of file G4MPImanager.hh.

messenger_

G4MPImessenger* G4MPImanager::messenger_

private

Definition at line 132 of file G4MPImanager.hh.

session_

G4MPIsession* G4MPImanager::session_

private

Definition at line 133 of file G4MPImanager.hh.

extra_worker_

G4VMPIextraWorker* G4MPImanager::extra_worker_

private

Definition at line 134 of file G4MPImanager.hh.

seed_generator_

G4VMPIseedGenerator* G4MPImanager::seed_generator_

private

Definition at line 137 of file G4MPImanager.hh.

status_

G4MPIstatus* G4MPImanager::status_

private

Definition at line 139 of file G4MPImanager.hh.

verbose_

G4int G4MPImanager::verbose_

private

Definition at line 141 of file G4MPImanager.hh.

is_master_

G4bool G4MPImanager::is_master_

private

Definition at line 144 of file G4MPImanager.hh.

is_slave_

G4bool G4MPImanager::is_slave_

private

Definition at line 145 of file G4MPImanager.hh.

is_extra_worker_

G4bool G4MPImanager::is_extra_worker_

private

Definition at line 146 of file G4MPImanager.hh.

rank_

G4int G4MPImanager::rank_

private

Definition at line 147 of file G4MPImanager.hh.

size_

G4int G4MPImanager::size_

private

Definition at line 148 of file G4MPImanager.hh.

world_size_

G4int G4MPImanager::world_size_

private

Definition at line 149 of file G4MPImanager.hh.

COMM_G4COMMAND_

MPI::Intracomm G4MPImanager::COMM_G4COMMAND_

private

Definition at line 152 of file G4MPImanager.hh.

processing_comm_

MPI_Comm G4MPImanager::processing_comm_

private

Definition at line 154 of file G4MPImanager.hh.

collecting_comm_

MPI_Comm G4MPImanager::collecting_comm_

private

Definition at line 156 of file G4MPImanager.hh.

all_comm_

MPI_Comm G4MPImanager::all_comm_

private

Definition at line 158 of file G4MPImanager.hh.

world_group_

MPI_Group G4MPImanager::world_group_

private

Definition at line 160 of file G4MPImanager.hh.

processing_group_

MPI_Group G4MPImanager::processing_group_

private

Definition at line 161 of file G4MPImanager.hh.

collecting_group_

MPI_Group G4MPImanager::collecting_group_

private

Definition at line 162 of file G4MPImanager.hh.

all_group_

MPI_Group G4MPImanager::all_group_

private

Definition at line 163 of file G4MPImanager.hh.

qfcout_

G4bool G4MPImanager::qfcout_

private

Definition at line 166 of file G4MPImanager.hh.

fscout_

std::ofstream G4MPImanager::fscout_

private

Definition at line 167 of file G4MPImanager.hh.

qinitmacro_

G4bool G4MPImanager::qinitmacro_

private

Definition at line 170 of file G4MPImanager.hh.

init_file_name_

G4String G4MPImanager::init_file_name_

private

Definition at line 171 of file G4MPImanager.hh.

qbatchmode_

G4bool G4MPImanager::qbatchmode_

private

Definition at line 172 of file G4MPImanager.hh.

macro_file_name_

G4String G4MPImanager::macro_file_name_

private

Definition at line 173 of file G4MPImanager.hh.

thread_id_

pthread_t G4MPImanager::thread_id_

private

Definition at line 176 of file G4MPImanager.hh.

fevents_in_master

G4int G4MPImanager::fevents_in_master = 0

private

Definition at line 177 of file G4MPImanager.hh.

fevents_in_slave

G4int G4MPImanager::fevents_in_slave = 0

private

Definition at line 178 of file G4MPImanager.hh.

master_weight_

G4double G4MPImanager::master_weight_

private

Definition at line 181 of file G4MPImanager.hh.

nof_extra_workers_

G4int G4MPImanager::nof_extra_workers_

private

Definition at line 182 of file G4MPImanager.hh.

---

The documentation for this class was generated from the following files:

• .doxygen/Doxymodules_parallel.h
• extended/parallel/MPI/source/include/G4MPImanager.hh
• extended/parallel/MPI/source/src/G4MPImanager.cc