Please read tips for efficient and successful posting and posting code
ROOT Version: 6.32.10
Platform: OSX
Compiler: 16.4 (apple silicon)
I am trying to filter branches from a large file as part of our open data project.
I get stymied by
“Fatal in TBufferFile::WriteFastArray: Not enough space left in the buffer (1GB limit). -1 elements is greater than the max left of 536870889”
BTW, the file I am testing on is < 1G but I have others that are 25 G.
I’ve tried T.SetMaxVirtualSize(size=1E9) as that worked in the distant past.
I’ve tried using TDataFrame - same issue.
How does one tell ROOT 6 to read in in decent size chunks and then make it possible to write them out?
Here is a script in python that reproduces the problem. It is reading from the MINERvA public data release.
import ROOT
from ROOT import TFile, TTree
import os
# an example of copytree.C
# I had trouble getting this to work on the latest versions of root
# but didn't try hard to diagnose the problem
# I run on a recent (c. 2024) version of root.
# From (Rik Gran)
# Minimal translated to Python by H. Schellman
infile = "root://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/minerva/persistent/OpenData/MediumEnergy_FHC/Data/Playlist1N/MasterAnaDev_data_AnaTuple_run00020408_Playlist.root"
f = TFile.Open(infile,"READONLY")
name = f.GetName()
DATA = "Data" in name
MC = "mc" in name
T = f.Get("MasterAnaDev")
if MC: TT = f.Get("Truth")
TM = f.Get("Meta")
# Needed to oversome some limitation
T.SetMaxVirtualSize(size=1E9)
if MC: TT.SetMaxVirtualSize(size=2E9)
TM.SetMaxVirtualSize(size=1E9)
#T.SetBranchStatus("mc_wgt_*",1)
if MC: TT.SetBranchStatus("mc_wgt_*",1)
# deactivate select branches
T.SetBranchStatus("lattice*",0)
T.SetBranchStatus("odlattice*",0)
T.SetBranchStatus("recoil_summed_energy*",0)
T.SetBranchStatus("recoil_data_fraction*",0)
T.SetBranchStatus("slice_hit_*",0)
T.SetBranchStatus("recoil*time_limit*",0)
T.SetBranchStatus("event_track_hit*",0)
T.SetBranchStatus("cluster_*",0)
T.SetBranchStatus("EnergyPoints*",0)
T.SetBranchStatus("VetoWall*",0)
T.SetBranchStatus("part_response*",0)
T.SetBranchStatus("part_response_total_recoil_passive_allNonMuonClusters_id",1)
T.SetBranchStatus("part_response_total_recoil_passive_allNonMuonClusters_od",1)
T.SetBranchStatus("*ichel*",0)
if MC: TT.SetBranchStatus("*ichel*",0)
T.SetBranchStatus("dEdX*",0)
T.SetBranchStatus("ExtraEnergyClusters*",0)
T.SetBranchStatus("prong_part*",0)
T.SetBranchStatus("proton_prong*",0)
T.SetBranchStatus("proton_track*",0)
T.SetBranchStatus("seco_prot*",0)
T.SetBranchStatus("sec_prot*",0)
T.SetBranchStatus("iso_prong*",0)
T.SetBranchStatus("gamma*",0)
T.SetBranchStatus("pi0*",0)
T.SetBranchStatus("disp*",0)
T.SetBranchStatus("blob_nuefuzz*",0)
T.SetBranchStatus("hadron_em*",0)
T.SetBranchStatus("proton_em*",0)
T.SetBranchStatus("phys_energy*",0)
T.SetBranchStatus("recoil_energy_*vtx*",0)
T.SetBranchStatus("clusters_found*",0)
T.SetBranchStatus("number_clusters*",0)
T.SetBranchStatus("shower_*",0)
T.SetBranchStatus("calibE_*",0)
T.SetBranchStatus("hadron_track_*",0)
T.SetBranchStatus("nonvtx_iso*",0)
T.SetBranchStatus("visE_*",0)
T.SetBranchStatus("Signal*",0)
T.SetBranchStatus("ConeEnergyVis",0)
T.SetBranchStatus("ExtraEnergyVis",0)
T.SetBranchStatus("Psi",0)
# One of the main neutron branches
# Notice I am adding back branches here in some cases
# after using a wildcard above
T.SetBranchStatus("MasterAnaDev_Blob*",1)
T.SetBranchStatus("MasterAnaDev_RecoPattern",1)
T.SetBranchStatus("MasterAnaDev_MCEnergyFrac*",0)
T.SetBranchStatus("MasterAnaDev_hadron*",0)
T.SetBranchStatus("MasterAnaDev_sec_prot*",0)
T.SetBranchStatus("MasterAnaDev_pi*",0)
T.SetBranchStatus("MasterAnaDev_prot*",0)
T.SetBranchStatus("MasterAnaDev_prot*",0)
T.SetBranchStatus("MasterAnaDev_sys*",0)
T.SetBranchStatus("numi*",0)
if MC: TT.SetBranchStatus("numi*",0)
T.SetBranchStatus("numi_pot*",1)
if MC: TT.SetBranchStatus("numi_pot*",1)
if MC: T.SetBranchStatus("truth_neutronInelast*",0)
if MC: TT.SetBranchStatus("truth_neutronInelastic*",0)
if MC: T.SetBranchStatus("truth_hadronReweight*",0)
if MC: TT.SetBranchStatus("truth_hadronReweight*",0)
if MC: T.SetBranchStatus("truth_muon_track_cluster*",0)
if MC: TT.SetBranchStatus("truth_muon_track_cluster*",0)
if MC: T.SetBranchStatus("truth_fuzz*",0)
if MC: TT.SetBranchStatus("truth_fuzz*",0)
if MC: T.SetBranchStatus("truth_gamma*",0)
if MC: T.SetBranchStatus("truth_prot*",0)
if MC: T.SetBranchStatus("truth_pi*",0)
if MC: T.SetBranchStatus("truth_muon_off_track*",0)
if MC: TT.SetBranchStatus("truth_gamma*",0)
if MC: TT.SetBranchStatus("truth_prot*",0)
if MC: TT.SetBranchStatus("truth_pi*",0)
if MC: TT.SetBranchStatus("truth_muon_off_track*",0)
T.SetBranchStatus("muon_track_cluster*",0)
T.SetBranchStatus("muon_fuzz_per_plane_r150*",0)
if MC: T.SetBranchStatus("mc_fr*",0)
if MC: TT.SetBranchStatus("mc_fr*",0)
T.SetBranchStatus("muon_thetaX_allNodes",0)
T.SetBranchStatus("muon_thetaY_allNodes",0)
T.SetBranchStatus("muon_theta_allNodes",0)
T.SetBranchStatus("muon_iso_blobs*",0)
file = TFile("outfile.root","recreate")
tree = T.CloneTree()
if MC: truth = TT.CloneTree()
meta = TM.CloneTree()
tree.Print()
if MC: truth.Print()
meta.Print()
file.Write(0,ROOT.TObject.kOverwrite)
file.Close()