Process killed

Hi ,
I have an executable of a C++ root macro that I compiled with ‘.L Select.C++’ and
then, out of root, using ‘g++ Select_C.so -o Select.exe’ I produced the exe file and I run it
on the shell.
The problem that I encounter is that when it runs on large number of events after event
number ~ 6 000 000 the process is killed and even the terminal window disappears!
I think there must be a problem with large size of the tree as I want to run on 20M events.
But I have tried several solutions (SetAutoSave, SetCircular,SetMaxSize,…) but it did not solve the problem.
How should I solve this problem?

Cheers,
Majid
p.s. the code is below: I run on root files whose names are in ChowderRootFiles.txt
and dump a selection of events with the same format to new tree and save them in output
root files.

#include <iostream>
#include <vector>
#include <utility>
#include <map>
#include <string>
using namespace std;
#include "/opt/5.14.00f-CMS3q/include/TH1F.h"
#include "/opt/5.14.00f-CMS3q/include/TTree.h"
#include "/opt/5.14.00f-CMS3q/include/TFile.h"
#include "/opt/5.14.00f-CMS3q/include/TSystem.h"
#include "/home/hashemim/CMSSW_1_6_8/src/Analysis/HWWAnalyzer/interface/myevent.h"
#include "/home/hashemim/CMSSW_1_6_8/src/Analysis/HWWAnalyzer/interface/LinkDef.h"

int main(){

  TFile *outputfile=new TFile("WnjOnly.root","recreate");
  TTree *t=new TTree("t","tree");
  //  t->SetAutoSave(1000000000);
  myevent *mout=new myevent;
  //  t->SetMaxTreeSize(1900000000);//A tree size of 5GB
  //  t->SetCircular(1000000000);
  t->Branch("myevent","myevent",&mout,256000,1);


  myevent *m=new myevent;
  TTree* rootTree;
  TBranch* branch;
  int ntot=0;
  int NumberOfFiles=1329;
  FILE *ffile=fopen("ChowderRootFiles.txt","r");

  for(int nf=0;nf<NumberOfFiles;nf++){
    char filename[20];
    fscanf(ffile,"%s",filename);
    TFile *file=TFile::Open(filename);
    rootTree = (TTree*)file->Get("t;1");
    int nev = int(rootTree->GetEntries());
    cout<<"Reading "<<filename<<" ;  number of entries is : "<<nev<<endl;
    branch = rootTree->GetBranch("myevent");
    branch->SetAddress(&m);

    int PID;//process id
    
  //================================================
  //event loop

    for(int i = 0; i < nev; i++){
      ntot++;
      cout<<"Reading "<<filename<<" ; event number "<<ntot<<endl;
      rootTree->GetEvent(i);
      
      PID=(m->ProcessId);
      
      //check if it is ttbar+nj (300x), W+nj (100x), Z+nj (200x)
      if(PID==1000 || PID==1001 || PID==1002 || PID==1003 || PID==1004 || PID==1005){
	mout=m;
	t->Fill();  

      }//check for PID
      
    }//loop over events
    file->Close();
  }//loop over files

  outputfile->Write();
  outputfile->Close();

  return 0;
}

Replace your statement;

file->Close(); by

delete file;
Rene

Hi Rene,
I have simplified the code to see where the problem comes from but still it is killed after
~7 000 000 events. I have attached the simplified code and I have already done what you
suggested but the problem exists yet.
This is a problem which can be not harmful for analyzing usual signal and background samples of ~1000 000 event size but we have soups of data which contain more than 10M
events.

Cheers,
Majid

#include <iostream>
#include <vector>
#include <utility>
#include <map>
#include <string>
using namespace std;
#include "/opt/5.14.00f-CMS3q/include/TH1F.h"
#include "/opt/5.14.00f-CMS3q/include/TTree.h"
#include "/opt/5.14.00f-CMS3q/include/TFile.h"
#include "/opt/5.14.00f-CMS3q/include/TSystem.h"
#include "/home/hashemim/CMSSW_1_6_8/src/Analysis/HWWAnalyzer/interface/myevent.h"
#include "/home/hashemim/CMSSW_1_6_8/src/Analysis/HWWAnalyzer/interface/LinkDef.h"

int main(){



  int ntot=0;
  int NumberOfFiles=1329;
  FILE *ffile=fopen("ChowderRootFiles.txt","r");
  char filename[20]={0};

  for(int nf=0;nf<NumberOfFiles;nf++){
    fscanf(ffile,"%s",filename);
    TFile *file=TFile::Open(filename);
    myevent *m=new myevent;
    TTree* rootTree= (TTree*)file->Get("t;1");
    TBranch* branch = rootTree->GetBranch("myevent");
    branch->SetAddress(&m);
    int nev = int(rootTree->GetEntries());
    cout<<"Reading "<<filename<<" ;  number of entries is : "<<nev<<endl;

    //int PID;//process id
    
  //================================================
  //event loop

    for(int i = 0; i < nev; i++){
      ntot++;
      cout<<"Reading "<<filename<<" ; event number "<<ntot<<endl;
      rootTree->GetEvent(i);
    }//loop over events

    delete m;
    //    delete branch;
    delete rootTree;
    delete file;
  }//loop over files


  return 0;
}

test.C (1.33 KB)

The only problem that I can see with your code is a memory leak in your class myevent (or classes used by myevent).
You should run valgrind and monitor the memory evolution with your number of events

Rene

Hi,
I have put test.C and all other needed files to compile the code below.
It can be compiled with root 5.18.
The datasets are in /castor/cern.ch/user/h/hashemim/
(Chowder1.root, Chowder2.root,…)
Do you have an idea where there could be the memory leak which causes the process
to be killed after ~10M events?
I tried valgrind but I did not see any problem with that.
Thanks,
Majid

#include <iostream>
#include <vector>
#include <utility>
#include <map>
#include <string>
using namespace std;
#include "TObject.h"
#include "TH1F.h"
#include "TTree.h"
#include "TFile.h"
#include "TSystem.h"
#include "myevent.h"
#include "LinkDef.h"

int main(){



  int ntot=0;
  int NumberOfFiles=18;
  FILE *ffile=fopen("ChowderRootFiles.txt","r");
  for(int nf=0;nf<NumberOfFiles;nf++){
    char filename[20];
    fscanf(ffile,"%s",filename);
    cout<<"Reading "<<filename<<endl;
    TFile *file=TFile::Open(filename);
    myevent *m=new myevent;
    TTree* rootTree=new TTree;
    TBranch* branch=new TBranch;
    rootTree = (TTree*)file->Get("t");
    int nev = int(rootTree->GetEntries());
    cout<<"number of entries is : "<<nev<<endl;
    branch = rootTree->GetBranch("myevent");
    branch->SetAddress(&m);
    
    for(int i = 0; i < nev; i++){
      ntot++;
      cout<<"event number "<<ntot<<endl;
      rootTree->GetEvent(i);
    }//loop over events
    //    delete branch;
    delete rootTree;
    delete m;
    delete file;
  }//loop over files
  
  
  return 0;
}

======================================================

#ifndef __MYEVENT_HH__
#define __MYEVENT_HH__
#include "TObject.h"
using namespace std;
#include <vector>
#include <map>
#include <utility>
#include "mytrack.h"
#include "myobject.h"

class myevent : public TObject {
  
  public :
    myevent(){;}
  ~myevent(){;}

  vector<myobject> RecJets;
  vector<myobject> RecElectrons;
  vector<myobject> RecMuons;
  vector<myobject> RecMet;
  vector<myobject> GenMet;
  vector<myobject> MCBjets;
  vector<myobject> MCMuons;
  vector<myobject> MCElectrons;
  vector<myobject> MCTaus;
  vector<myobject> MCNus;
  vector<myobject> GenParticle;
  vector<myobject> Vertex; 
  vector<mytrack>  AllTracks;
  map<string,int> HLT;
  double weight;
  double KFactor;
  int ProcessId;
  double PtHat;
  bool PreselectionMarker;

  void Clean(){
    RecJets.clear();
    RecElectrons.clear();
    RecMuons.clear();
    RecMet.clear();
    GenMet.clear();
    MCBjets.clear();
    MCMuons.clear();
    MCElectrons.clear();
    MCTaus.clear();
    MCNus.clear();
    GenParticle.clear();
    Vertex.clear();
    AllTracks.clear();
    HLT.clear();
    weight=1;
    KFactor=1;
    ProcessId=0;
    PtHat=0;
    PreselectionMarker=false;
  }

  private :

  ClassDef (myevent,1)
};
#endif

============================================================

#ifndef __MYOBJECT_HH__
#define __MYOBJECT_HH__
#include "TROOT.h"
#include "TObject.h"
using namespace std;
#include <vector>
#include "mytrack.h"
class myobject : public TObject {
public :
  myobject()
  {
    pt=0;
    eta=0;
    phi=0;
    charge=0;
    E=0;
    px=0;
    py=0;
    pz=0;
    corrpt=0;
    corrpx=0;
    corrpy=0;
    corrphi=0;
    DepositR03Ecal=0;
    DepositR03Hcal=0;
    DepositR03TrackerOfficial=0;
    DepositR03TrackerMine=0;
    alpha=0;
    ID_Is_Ok=0;
    CaloIsolation=0;
    TrackerIsolation=0;
  }
    ~myobject(){;}

  double pt,eta,phi,charge,E,px,py,pz;
  double corrpt,corrpx,corrpy,corrphi;
  //  vector<mytrack> tracks;
  double DepositR03Ecal;
  double DepositR03Hcal;
  double DepositR03TrackerOfficial;
  double DepositR03TrackerMine;
  double alpha;
  bool ID_Is_Ok;
  bool CaloIsolation;
  bool TrackerIsolation;
  string name;

  ClassDef (myobject,1)
};
#endif

========================================================

#ifndef __MYTRACK_HH__
#define __MYTRACK_HH__
#include "TROOT.h"
#include "TObject.h"
#include <vector>
class mytrack : public TObject {
public :
  mytrack(){;}
  ~mytrack(){;}
  double px,py,pz,pt,eta,phi,charge,E;
  double ipx,ipy;
  double iperr_x,iperr_y;
  double Vz,Chi2,Ndof,VHits;
  ClassDef (mytrack,1)
};
#endif

=========================================

#ifdef __CINT__
#pragma link off all globals;
#pragma link off all classes;
#pragma link off all functions;
#pragma link C++ nestedclass;
#pragma link C++ class mytrack+;
#pragma link C++ class vector<mytrack>;
#pragma link C++ class vector<mytrack>::iterator;
#pragma link C++ function operator==(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator!=(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator<=(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator>=(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator<(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator>(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator-(const vector<mytrack>::iterator&,const vector<mytrack>::iterator&); 
#pragma link C++ function operator+(const vector<mytrack>::iterator::difference_type,const vector<mytrack>::iterator&); 
#pragma link C++ class myobject+;
#pragma link C++ class vector<myobject>;
#pragma link C++ class vector<myobject>::iterator;
#pragma link C++ function operator==(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator!=(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator<=(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator>=(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator<(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator>(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator-(const vector<myobject>::iterator&,const vector<myobject>::iterator&); 
#pragma link C++ function operator+(const vector<myobject>::iterator::difference_type,const vector<myobject>::iterator&); 
#pragma link C++ class myevent+;
#endif