How to get tree branch variables for a specific entry based on the thresholds from the ROC curve?

TMVA
1

Hi, ROOT experts,

I have a script to make the ROC curve out of the TMVA output file by using the cuts:

import numpy as np
import ROOT
from ROOT import TFile, TTree, TCanvas, TGraph, TPad, TLegend

f = TFile.Open("factoryOutputFile.root")
t = f.Get("dataLoader/TestTree")

nSigTot = t.Draw("PyTorch", "classID==0")
nBkgTot = t.Draw("PyTorch", "classID==1")

gr = TGraph()

cutValues = np.array([])
cut = 0
while cut <= 0.96:
    cutValues = np.append(cutValues, cut)
    cut += 0.01
while cut > 0.96 and cut <= 1.0:
    cutValues = np.append(cutValues, cut)
    cut += 0.001

for cut in cutValues:
    cutSig = f"classID==0 && PyTorch > {cut}"
    cutBkg = f"classID==1 && PyTorch > {cut}"

    sigCount = t.GetEntries(cutSig)
    bkgCount = t.GetEntries(cutBkg)

    eff = sigCount / nSigTot
    rej = 1 - bkgCount / nBkgTot
    gr.SetPoint(gr.GetN(), eff, rej)

canvas = TCanvas("c1", "Signal efficiency vs. Background rejection", 10, 10, 850, 500)
canvas.cd()
ROOT.gStyle.SetOptStat(0)
ROOT.gPad.SetLeftMargin(0.15)

grid = TPad("grid", "", 0, 0, 1, 1)
grid.Draw()
grid.cd()
grid.SetGrid()

gr.SetTitle("Signal efficiency vs. Background rejection")
gr.GetXaxis().SetTitle("Signal efficiency (Sensitivity)")
gr.GetXaxis().SetRangeUser(0.92,1.0015)
gr.GetYaxis().SetTitle("Background rejection (Specificity)")
gr.GetYaxis().SetRangeUser(0.9,1.001)
gr.SetLineWidth(2)
gr.SetLineColor(4)
gr.Draw("AL")

leg = TLegend(0.15, 0.15, 0.35, 0.3)
leg.SetHeader("MVA Method", "")
leg.AddEntry(gr, "PyTorch_CNN", "l");
leg.Draw()

canvas.Print("cuts.pdf", "pdf")

Here is the plot:
roc.pdf (14.3 KB)

Now I am trying to find a way to get the event numbers added by loader.AddSpectator("event_number") for the false positives at a certain threshold because I want to see what background events didn’t get rejected by the classifier; for example, when the sensitivity equals 0.97 and the background rejection equals 0.9989 by looking at the ROC curve, how do I do that? I have event_number in the TestTree, hope you can give me some guidance.

Thank you!

2

I guess @moneta can help.

3

Hi @couet
Thank you for tagging @moneta for me.

4

Hi @moneta,
Could you please help me with this question? That would be highly appreciated, thank you!

5

I’ve figured it out:

import numpy as np
import ROOT
from ROOT import TFile, TTree, TCanvas, TGraph, TPad, TLegend

f = TFile.Open("factoryOutputFile.root")
t = f.Get("dataLoader/TestTree")

nSigTot = t.Draw("PyTorch", "classID==0")
nBkgTot = t.Draw("PyTorch", "classID==1")

gr = TGraph()

cutValues = np.array([])
cut = 0
while cut <= 0.96:
    cutValues = np.append(cutValues, cut)
    cut += 0.01
while cut > 0.96 and cut <= 1.0:
    cutValues = np.append(cutValues, cut)
    cut += 0.001

nFalsePositiveEvents = 0
for cut in cutValues:
    cutSig = f"classID==0 && PyTorch > {cut}"
    cutBkg = f"classID==1 && PyTorch > {cut}"

    sigCount = t.GetEntries(cutSig)
    bkgCount = t.GetEntries(cutBkg)

    eff = sigCount / nSigTot
    rej = 1 - bkgCount / nBkgTot
    gr.SetPoint(gr.GetN(), eff, rej)

    if eff < 0.9705 and eff > 0.9695:
        print(f"*** Signal Efficiency: {eff}")
        for i in range(bkgCount):
            t.GetEntry(i)
            if t.classID == 1:
                nFalsePositiveEvents += 1
                run_number = int(t.run_number)
                event_number = int(t.event_number)
                print(f"Run {run_number}   Event {event_number}")

print(f"Number of False Positive Events: {nFalsePositiveEvents}")

canvas = TCanvas("c1", "Signal efficiency vs. Background rejection", 10, 10, 850, 500)
canvas.cd()
ROOT.gStyle.SetOptStat(0)
ROOT.gPad.SetLeftMargin(0.15)

grid = TPad("grid", "", 0, 0, 1, 1)
grid.Draw()
grid.cd()
grid.SetGrid()

gr.SetTitle("Signal efficiency vs. Background rejection")
gr.GetXaxis().SetTitle("Signal efficiency (Sensitivity)")
gr.GetXaxis().SetRangeUser(0.92,1.0015)
gr.GetYaxis().SetTitle("Background rejection (Specificity)")
gr.GetYaxis().SetRangeUser(0.9,1.001)
gr.SetLineWidth(2)
gr.SetLineColor(4)
gr.Draw("AL")

leg = TLegend(0.15, 0.15, 0.35, 0.3)
leg.SetHeader("MVA Method", "")
leg.AddEntry(gr, "PyTorch_CNN", "l");
leg.Draw()

canvas.Print("cuts.pdf", "pdf")
6

Update: (There’s a small mistake in the code above)

import numpy as np
import ROOT
from ROOT import TFile, TTree, TCanvas, TGraph, TPad, TLegend

f = TFile.Open("factoryOutputFile.root")
t = f.Get("dataLoader/TestTree")

nSigTot = t.Draw("PyTorch", "classID==0")
nBkgTot = t.Draw("PyTorch", "classID==1")

gr = TGraph()

cutValues = np.array([])
cut = 0
while cut <= 0.96:
    cutValues = np.append(cutValues, cut)
    cut += 0.01
while cut > 0.96 and cut <= 1.0:
    cutValues = np.append(cutValues, cut)
    cut += 0.001

nFalsePositiveEvents = 0
for cut in cutValues:
    cutSig = f"classID==0 && PyTorch > {cut}"
    cutBkg = f"classID==1 && PyTorch > {cut}"

    sigCount = t.GetEntries(cutSig)
    bkgCount = t.GetEntries(cutBkg)

    eff = sigCount / nSigTot
    rej = 1 - bkgCount / nBkgTot
    gr.SetPoint(gr.GetN(), eff, rej)

    if eff < 0.9705 and eff > 0.9695:
        print(f"*** Signal Efficiency: {eff}")
        i = 0
        while nFalsePositiveEvents < bkgCount:
            t.GetEntry(i)
            if t.classID == 1 and t.PyTorch > cut:
                nFalsePositiveEvents += 1
                run_number = int(t.run_number)
                event_number = int(t.event_number)
                print(f"Run {run_number}   Event {event_number}")
            i += 1

print(f"Number of False Positive Events: {nFalsePositiveEvents}")

canvas = TCanvas("c1", "Signal efficiency vs. Background rejection", 10, 10, 850, 500)
canvas.cd()
ROOT.gStyle.SetOptStat(0)
ROOT.gPad.SetLeftMargin(0.15)

grid = TPad("grid", "", 0, 0, 1, 1)
grid.Draw()
grid.cd()
grid.SetGrid()

gr.SetTitle("Signal efficiency vs. Background rejection")
gr.GetXaxis().SetTitle("Signal efficiency (Sensitivity)")
gr.GetXaxis().SetRangeUser(0.92,1.0015)
gr.GetYaxis().SetTitle("Background rejection (Specificity)")
gr.GetYaxis().SetRangeUser(0.9,1.001)
gr.SetLineWidth(2)
gr.SetLineColor(4)
gr.Draw("AL")

leg = TLegend(0.15, 0.15, 0.35, 0.3)
leg.SetHeader("MVA Method", "")
leg.AddEntry(gr, "PyTorch_CNN", "l");
leg.Draw()

canvas.Print("cuts.pdf", "pdf")