#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "TrackHeed.hh" #include "ComponentAnsys123.hh" #include "ViewField.hh" #include "MediumMagboltz.hh" #include "Sensor.hh" #include "AvalancheMicroscopic.hh" #include "AvalancheMC.hh" #include "Random.hh" #include "Plotting.hh" #include "RandomEngineMPIServer.hh" #include #include #include #include "read_card.h" #define DIETAG 1 #define WORKTAG 2 #define RESULT_TAG 3 #define RESULTE_TAG 4 #define RESULTI_TAG 5 using namespace Garfield; using namespace std; typedef struct EventParam_s { int eventID; double x0; double y0; double z0; double t0; double e0; } EventParam; typedef struct EventResult_s { int eventID; int np; int ne; int ni; int tupleSize; } EventResult; MPI_Datatype mpi_event_param_type; MPI_Datatype mpi_event_result_type; /* Local functions */ static void master(int argc, char** argv, int size, int nEvents); static void slave(int myrank, int size, int random_server); static void random_number_server(void); void create_mpi_event_param_type(void); void create_mpi_event_result_type(void); int main(int argc, char **argv) { int myrank, size; int nEvents = 0; /* Initialize MPI */ MPI_Init(&argc, &argv); /* Find out my identity in the default communicator */ MPI_Comm_rank(MPI_COMM_WORLD, &myrank); MPI_Comm_size(MPI_COMM_WORLD, &size); int res = read_card(argc, argv); if(!res) { std::cout << "Error while reading the card file.\n" ; return 1; } nEvents = param_num_events; if(myrank==0) cout << "NUMBER OF EVENTS: " << nEvents << endl; if(size<3) { cerr << "Error: Requries at least 3 processes (1 master, 1 random_server, 1 or more slaves).\n"; MPI_Finalize(); return 1; } if(nEvents < size-2) { cerr << "Error: Number of events should not be less than number of slaves.\n"; MPI_Finalize(); return 1; } // Create a MPI type for EventResult structure create_mpi_event_param_type(); // Create a MPI type for EventResult structure create_mpi_event_result_type(); if (myrank == 0) { master(argc, argv, size, nEvents); } else if (myrank == size-1) { random_number_server(); } else { slave(myrank, size, size-1); } /* Shut down MPI */ MPI_Finalize(); return 0; } void random_number_server(void) { RandomEngineMPIServer server; cout << "Random Server: starting the random number generator" << endl; server.Run(); cout << "Random Server: random number generator killed" << endl; } // Dimensions of the GEM //const double pitch = 0.014; //const double kapton = 50.e-4; //const double metal = 5.e-4; //const double outdia = 70.e-4; //const double middia = 50.e-4; // Field map ComponentAnsys123* fm; // Gas MediumMagboltz* gas; // Sensor Sensor* sensor; // Microsopic avalanche AvalancheMicroscopic* aval; // Drift AvalancheMC* drift; //Track TrackHeed* track; void master(int argc, char** argv, int size, int nEvents) { TApplication app("app", &argc, argv); // plottingEngine.SetDefaultStyle(); // Histograms int nBinsGain = 100; double gmin = 0.; double gmax = 100.; //TH1F* hElectrons = new TH1F("hElectrons", "Number of electrons", // nBinsGain, gmin, gmax); //TH1F* hIons = new TH1F("hIons", "Number of ions", // nBinsGain, gmin, gmax); int nBinsChrg = 100; //TH1F* hChrgE = new TH1F("hChrgE", "Electrons on plastic", // nBinsChrg, -0.5e4 * kapton, 0.5e4 * kapton); //TH1F* hChrgI = new TH1F("hChrgI", "Ions on plastic", // nBinsChrg, -0.5e4 * kapton, 0.5e4 * kapton); double sumIonsTotal = 0.; double sumIonsDrift = 0.; double sumIonsPlastic = 0.; double sumElectronsTotal = 0.; double sumElectronsPlastic = 0.; double sumElectronsUpperMetal = 0.; double sumElectronsLowerMetal = 0.; double sumElectronsTransfer = 0.; double sumElectronsOther = 0.; vector v_chrgE, v_chrgI; vector v_xe1, v_ye1, v_ze1, v_te1, v_e1; vector v_xe2, v_ye2, v_ze2, v_te2, v_e2; EventResult res; MPI_Status status, status2; ofstream out; out.open("progress.dat"); // Event parameters structure EventParam param; // Load the root file containing the random energies //TFile *f = new TFile("driftEdep.root"); //TTree *t = (TTree*)f->Get("drift_energy"); //TH1F *initialE = new TH1F("initialE","initialE",100,0.0,0.1); TFile *f = new TFile("ntuples.root", "RECREATE"); TNtuple* ntuple = new TNtuple("ntuple","","xe1:ye1:ze1:te1:e1:xe2:ye2:ze2:te2:e2"); TNtuple* ntuple1 = new TNtuple("ntuple1", "", "gain:np:ne:ni"); int iEvent=0, iCompleted=0; for(; iEventGetRandom(); MPI_Send(¶m, 1, mpi_event_param_type, slave_id, WORKTAG, MPI_COMM_WORLD); } while(iCompletedGetRandom(); MPI_Send(¶m, 1, mpi_event_param_type, status.MPI_SOURCE, WORKTAG, MPI_COMM_WORLD); //MPI_Send(&iEvent, 1, MPI_INT, status.MPI_SOURCE, WORKTAG, MPI_COMM_WORLD); iEvent++; } } f->Write(); out.close(); int dummy=0; // Send DIETAG to all slaves for(int i=1; i<=size-2; i++) MPI_Send(&dummy, 1, MPI_INT, i, DIETAG, MPI_COMM_WORLD); // Terminate the random number server MPI_Send(&dummy, 1, MPI_INT, size-1, RandomEngineMPI::KILL, MPI_COMM_WORLD ); double fFeedback = 0.; if (sumIonsTotal > 0.) fFeedback = sumIonsDrift / sumIonsTotal; //std::cout << "Fraction of ions drifting back: " << fFeedback << "\n"; //const double neMean = hElectrons->GetMean(); //std::cout << "Mean number of electrons: " << neMean << "\n"; //const double niMean = hIons->GetMean(); //std::cout << "Mean number of ions: " << niMean << "\n"; /*TFile f("histos.root","new"); TH1F h1("hgaus","histo from a gaussian",100,-3,3); h1.FillRandom("gaus",10000); h1.Write();*/ const bool plotHistogram = false; if (plotHistogram) { TCanvas* cH = new TCanvas("cH", "Histograms", 800, 700); cH->Divide(2, 2); cH->cd(1); //hElectrons->Draw(); cH->cd(2); //hIons->Draw(); cH->cd(3); //hChrgE->Draw(); cH->cd(4); //hChrgI->Draw(); cH->SaveAs("histogram.eps"); TFile f("histos.root","new"); cH->Write(); } //app.Run(kTRUE); } void do_work(const EventParam& param, EventResult& res, vector& v_xe1, vector& v_ye1, vector& v_ze1, vector& v_te1, vector& v_e1, vector& v_xe2, vector& v_ye2, vector& v_ze2, vector& v_te2, vector& v_e2); void slave(int myrank, int size, int random_server) { const bool debug = true; // Load the field map. fm = new ComponentAnsys123(); const std::string efile = param_mesh_dir + "/ELIST.lis"; const std::string nfile = param_mesh_dir + "/NLIST.lis"; const std::string mfile = param_mesh_dir + "/MPLIST.lis"; const std::string sfile = param_mesh_dir + "/PRNSOL.lis"; fm->Initialise(efile, nfile, mfile, sfile, "mm"); fm->EnableMirrorPeriodicityX(); fm->EnableMirrorPeriodicityY(); fm->PrintRange(); /* if(param_opt_cache_boundingboxes) fm->EnableCachingOfBoundingBoxes(); else fm->DisableCachingOfBoundingBoxes(); if(param_opt_search_tetratree) fm->EnableTetrahedralTreeForElementSearch(); else fm->DisableTetrahedralTreeForElementSearch(); if(param_opt_search_neighbors) fm->EnableSearchThroughNeighbors(); else fm->DisableSearchThroughNeighbors(); */ // Dimensions of the GEM const double pitch = param_dim_pitch; const double kapton = param_dim_kapton; const double metal = param_dim_metal; const double outdia = param_dim_outdia; const double middia = param_dim_middia; // Setup the gas. gas = new MediumMagboltz(); gas->SetComposition(param_gas1_name, param_gas1_comp, param_gas2_name, param_gas2_comp, param_gas3_name, param_gas3_comp); gas->SetTemperature(param_gas_temperature); gas->SetPressure(param_gas_pressure); //gas->EnableDebugging(); gas->Initialise(); gas->DisableDebugging(); // Set the Penning transfer efficiency. const double rPenning = param_gas_penning; const double lambdaPenning = 0.; gas->EnablePenningTransfer(rPenning, lambdaPenning, "ar"); // Load the ion mobilities. gas->LoadIonMobility(param_ion_filepath); // Associate the gas with the corresponding field map material. const int nMaterials = fm->GetNumberOfMaterials(); for (int i = 0; i < nMaterials; ++i) { const double eps = fm->GetPermittivity(i); if (fabs(eps - 1.) < 1.e-3) fm->SetMedium(i, gas); } fm->PrintMaterials(); // Create the sensor. sensor = new Sensor(); sensor->AddComponent(fm); sensor->SetArea(param_sensor_x1, param_sensor_y1, param_sensor_z1, param_sensor_x2, param_sensor_y2, param_sensor_z2); aval = new AvalancheMicroscopic(); aval->SetSensor(sensor); //aval->EnableDebugging(); drift = new AvalancheMC(); drift->SetSensor(sensor); drift->SetDistanceSteps(2.e-4); // Create the track double momentum = 5e+08; track = new TrackHeed(); track->SetSensor(sensor); track->SetParticle("mu-"); track->DisableDeltaElectronTransport(); //ONLY DELTA ELECTRONS PRODUCED track->SetMomentum(momentum); track->EnableDebugging(); /* const double smear = pitch / 2.; double x0 = -smear + RndmUniform() * smear; double y0 = -smear + RndmUniform() * smear; double z0 = param_sensor_z2 - .001; double t0i = 0.0; double e0 = 0.1; double dx0i = 0.0, dy0i = 0., dz0i = -1.; track->NewTrack(x0, y0, z0, t0i, dx0i, dy0i, dz0i); */ EventParam param = {0}; EventResult res = {0}; vector v_chrgE, v_chrgI; vector v_xe1, v_ye1, v_ze1, v_te1, v_e1; vector v_xe2, v_ye2, v_ze2, v_te2, v_e2; int eventID; MPI_Status status; while(1) { /* Receive a message from the master */ MPI_Recv(¶m, 1, mpi_event_param_type, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &status); /* Check the tag of the received message. */ if (status.MPI_TAG == DIETAG) { break; } else if(status.MPI_TAG == WORKTAG) { /* Do the work */ // v_chrgE.clear(); //v_chrgI.clear(); v_xe1.clear(); v_ye1.clear(); v_ze1.clear(); v_te1.clear(); v_e1.clear(); v_xe2.clear(); v_ye2.clear(); v_ze2.clear(); v_te2.clear(); v_e2.clear(); res = {0}; do_work(param, res, v_xe1, v_ye1, v_ze1, v_te1, v_e1, v_xe2, v_ye2, v_ze2, v_te2, v_e2); /* Send the result back */ MPI_Send(&res, 1, mpi_event_result_type, 0, RESULT_TAG, MPI_COMM_WORLD); MPI_Send(&v_xe1[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_ye1[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_ze1[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_te1[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_e1[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_xe2[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_ye2[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_ze2[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_te2[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); MPI_Send(&v_e2[0], res.tupleSize, MPI_DOUBLE, 0, RESULTE_TAG, MPI_COMM_WORLD); } else { cerr << "Slave: Unknow MPI tag" << endl; } } } void do_work(const EventParam& param, EventResult& res, vector& v_xe1, vector& v_ye1, vector& v_ze1, vector& v_te1, vector& v_e1, vector& v_xe2, vector& v_ye2, vector& v_ze2, vector& v_te2, vector& v_e2) { // Randomize the initial position. res = { 0 }; res.eventID = param.eventID; // The initial impact position of the incoming ionising track // Weston Cadena const double pitch = param_dim_pitch; const double smear = pitch / 2.; double track_x = -smear + RndmUniform() * smear; double track_y = -smear + RndmUniform() * smear; double track_z = param_sensor_z2 - .001; double t0 = 0.0; double e0 = 0.1; double tMin = 0.0; int Total_Number_Electrons = 0; int Total_Electrons_Along_Track = 0; int Total_Energy_Loss = 0.; int Total_Number_Clusters; // Momentum direction of incoming track double track_dx = 0.0; double track_dy = 0.0; // double track_dz = -1.0; //Track is downstream //GetCluster variables double xcls, ycls, zcls, tcls, e, extra; int nclus = 0; //GetElectron variables double xele, yele, zele, tele, eele, dxele, dyele, dzele; //GENERAL TRACK ANSWERS int npe_t = 0; int ne_t = 0; int ni_t = 0; int myrank; MPI_Comm_rank(MPI_COMM_WORLD, &myrank); track->NewTrack(track_x, track_y, track_z, tMin, track_dx, track_dy, track_dz); std::cout << myrank << ": NEW TRACK CALLED" << std::endl; while (track->GetCluster(xcls, ycls, zcls, tcls, nclus, e, extra)) { //number of primary electrons in collision npe_t += nclus; //Getting Information from each primary electron for (int iclus = 1; iclus <= nclus; iclus++) { track->GetElectron(iclus, xele, yele, zele, tele, eele, dxele, dyele, dzele); double x0 = xele; double y0 = yele; double z0 = zele; double t0 = tele; double e0 = eele; double dx0 = dxele; double dy0 = dyele; double dz0 = dzele; //Avalanche aval->AvalancheElectron(x0, y0, z0, t0, e0, dx0, dy0, dz0); int ne = 0, ni = 0; aval->GetAvalancheSize(ne, ni); ne_t += ne; ni_t += ni; std::cout << std::fixed << std::setprecision(7) << myrank << ": Elecctron: X0=" << xele << " Y0=" << yele << " Z0=" << zele << " t0=" << tele << " e0=" << eele; std::cout << std::fixed << std::setprecision(7) << " DX0=" << dxele << " DY0=" << dyele << " DZ0=" << dzele << std::endl; std::cout << myrank << ": Track " << param.eventID << ": Avalanche Size: ne=" << ne << ", ni=" << ni << std::endl; // Getting Endpoints const int np = aval->GetNumberOfElectronEndpoints(); //int np = 1; double xe1, ye1, ze1, te1, e1; double xe2, ye2, ze2, te2, e2; double xi1, yi1, zi1, ti1; double xi2, yi2, zi2, ti2; int status; for (int j = np; j--;) { aval->GetElectronEndpoint(j, xe1, ye1, ze1, te1, e1, xe2, ye2, ze2, te2, e2, status); v_xe1.push_back(xe1); v_ye1.push_back(ye1); v_ze1.push_back(ze1); v_te1.push_back(te1); v_e1.push_back(e1); v_xe2.push_back(xe2); v_ye2.push_back(ye2); v_ze2.push_back(ze2); v_te2.push_back(te2); v_e2.push_back(e2); drift->DriftIon(xe1, ye1, ze1, te1); //drift->GetIonEndpoint(0, xi1, yi1, zi1, ti1, //xi2, yi2, zi2, ti2, status); } // endpoints loop } // iclus loop } // while loop res.np = npe_t; res.ne = ne_t; res.ni = ni_t; res.tupleSize = v_xe1.size(); std::cout << myrank << ": ##TOTAL## TRACK" << param.eventID << ": Avalanche Size: ne_t=" << ne_t << ", ni_t=" << ni_t << std::endl; } void create_mpi_event_param_type(void) { EventParam param; const int nitems=2; int blocklengths[2] = {1,1}; MPI_Datatype old_types[2] = {MPI_INT,MPI_DOUBLE}; MPI_Aint displ[2]; MPI_Get_address(¶m.eventID, &displ[0]); MPI_Get_address(¶m.e0, &displ[1]); for(int i=nitems-1; i>=0; i--) displ[i] -= displ[0]; MPI_Type_create_struct(nitems, blocklengths, displ, old_types, &mpi_event_param_type); MPI_Type_commit(&mpi_event_param_type); } void create_mpi_event_result_type(void) { EventResult res; const int nitems=5; int blocklengths[5] = {1,1,1,1,1}; MPI_Datatype old_types[5] = {MPI_INT,MPI_INT,MPI_INT,MPI_INT,MPI_INT}; MPI_Aint displ[5]; MPI_Get_address(&res.eventID, &displ[0]); MPI_Get_address(&res.np, &displ[1]); MPI_Get_address(&res.ne, &displ[2]); MPI_Get_address(&res.ni, &displ[3]); MPI_Get_address(&res.tupleSize, &displ[4]); for(int i=nitems-1; i>=0; i--) displ[i] -= displ[0]; MPI_Type_create_struct(nitems, blocklengths, displ, old_types, &mpi_event_result_type); MPI_Type_commit(&mpi_event_result_type); }