Limit number of entries in the TTree draw in JSROOT

ROOT
1

Dear JSROOT experts,

I am trying to draw TGraph form the large (~1GB) root file using TTree draw functionality in jsroot. Since the file is very large and the requested branches (Amplitude) contains ~ 1.2e+08 entries, I would like to speed up the drawing by limiting number of entries used to build the graph. According to the docs I tried setting entries:10000 in the draw command like this:

            let tree_cur = await cur_file.readObject("CALIWAVE");
            let tree_ref = await ref_file.readObject("CALIWAVE");
            
            var draw_cmd = "Amplitude::(barrel_ec=="+ data["barrel_ec"] +" && pos_neg=="+ data["pos_neg"] +" && FT=="+ data["ft"] +" && slot=="+ data["sl"] +" && channel=="+ data["ch"] +" && corrUndo==0);entries:10000;graph;drawopt:LP"
            let g_cur_tree = await JSROOT.draw(plot_id_tree, tree_cur, draw_cmd);
            let g_ref_tree = await JSROOT.draw(plot_id_tree, tree_ref, draw_cmd);

however this does not seem to have any effect on the drawing time - it still takes several minutes to fully load both graphs. So my question is if entries option is supposed to work with graph and in API, and if there are better ways to control the amount of data being processed by the TTree draw, for example by loading only part of the root file (since as I understand this is done in bunches (GET requests) which are loaded synchronously)…
Thanks for all your ideas and help.

Best,
Dominik

2

Hello @dondo, I am sure @linev can answer your question.

3

Hi Dominik,

JSROOT handle drawing into the graph exactly the same way as all other kinds of drawings.
Only required range of entries for selected branches are requested from the server.

I have a number of questions:
Which JSROOT version you are using?
Did you try to specify minimal number of entries - let say 10?
Did you try to use simple draw expression without any condition?

I have small comment about your code.

Are you using same plot_id_tree for both drawings? This may not work.

You using await statement for each draw. You can perform drawing in parallel - like:

let pr1 = JSROOT.draw(plot_id_tree, tree_cur, draw_cmd);
let pr2 = JSROOT.draw(plot_id_tree, tree_ref, draw_cmd);
await Promise.all([pr1, pr2]);

Regards,
Sergey

4

Hi Sergey,

thanks for the prompt reply. Version I use is the latest one ( 7.8.2 26/03/2025). I tried the tests you suggested and I already see some improvement in the drawing time. Reason why I use async for the draw calls is that with the obtained graphs I want to produce multigraph at the end (see full function below).

async function drawPulseShape(data) {
    var plot_id_tree_cur = "shape_plot_tree_cur_"+data["part_side"]+"_"+data["ft"]+"_"+data["sl"]+"_"+data["ch"]+"_"+data["gain"];
    var plot_id_tree_ref = "shape_plot_tree_ref_"+data["part_side"]+"_"+data["ft"]+"_"+data["sl"]+"_"+data["ch"]+"_"+data["gain"];
    var plot_id = "shape_plot_"+data["part_side"]+"_"+data["ft"]+"_"+data["sl"]+"_"+data["ch"]+"_"+data["gain"];
    var shape_plot = document.getElementById(plot_id);
    if (shape_plot.getAttribute("name") === "not-computed"){
        if (data["cur_root_files"].length > 1) {
            console.log("More than one root file found for pulse shape for current campaign, will take only first!")
        }
        if (data["ref_root_files"].length > 1) {
            console.log("More than one root file found for pulse shape for reference campaign, will take only first!")
        }
        let cur_file = await JSROOT.openFile(data["cur_root_files"][0]);
        let ref_file = await JSROOT.openFile(data["ref_root_files"][0]);
        JSROOT.settings.ApproxTextSize = true;
        JSROOT.gStyle.fOptLogy = 0;
        try {
            shape_plot.innerHTML = "";
            shape_plot.style.width = "800px";
            shape_plot.style.height = "600px";
            let tree_cur = await cur_file.readObject("CALIWAVE");
            let tree_ref = await ref_file.readObject("CALIWAVE");
            
            var draw_cmd = "Amplitude::(barrel_ec=="+ data["barrel_ec"] +" && pos_neg=="+ data["pos_neg"] +" && FT=="+ data["ft"] +" && slot=="+ data["sl"] +" && channel=="+ data["ch"] +" && corrUndo==0);entries:1000;graph;drawopt:LP"
            let g_cur_tree = await JSROOT.draw(plot_id_tree_cur, tree_cur, draw_cmd);
            let g_ref_tree = await JSROOT.draw(plot_id_tree_ref, tree_ref, draw_cmd);
            
            let npoints = g_cur_tree.draw_object.fMaxSize;
            let npoints_ref = g_ref_tree.draw_object.fMaxSize;

            let g_cur = JSROOT.createTGraph(npoints, g_cur_tree.draw_object.fX, g_cur_tree.draw_object.fY);
            let g_ref = JSROOT.createTGraph(npoints_ref, g_ref_tree.draw_object.fX, g_ref_tree.draw_object.fY);
            
            g_cur.fName = "Current"
            g_ref.fName = "Reference"
            g_cur.fLineColor = 2;
            g_cur.fMarkerColor = 2;
            g_cur.fMarkerSize = 1;
            g_cur.fMarkerStyle = 22;
            g_ref.fMarkerColor = 4;
            g_ref.fLineColor = 4;
            g_ref.fMarkerSize = 1;
            g_ref.fMarkerStyle = 22;

            let text_size = 0.028;

            let text_def_camp = JSROOT.create("TPaveText");
            g_cur.fFunctions.Add(text_def_camp);
            Object.assign(text_def_camp, { fX1NDC: 0.3, fY1NDC: 0.7, fX2NDC: 0.9, fY2NDC: 0.8 });
            text_def_camp.AddText(data["part_side"]+", FT = "+data["ft"]+", SL = "+data["sl"]+", CH = "+data["ch"]+", Gain = "+data["gain"]);
            text_def_camp.fTextColor = 1;
            text_def_camp.fTextAlign = 11;
            for (let i = 0; i < text_def_camp.fLines.arr.length; i++) {
                text_def_camp.fLines.arr[i].fTextSize = text_size;
            }
            
            let text_def = JSROOT.create("TPaveText");
            g_cur.fFunctions.Add(text_def);
            Object.assign(text_def, { fX1NDC: 0.3, fY1NDC: 0.65, fX2NDC: 0.9, fY2NDC: 0.75 });
            text_def.AddText("Campaign "+data["id"]+" from "+data["cur_date"]+" run = "+data["cur_run_id"][0]);
            text_def.fTextColor = 2;
            text_def.fTextAlign = 11;
            for (let i = 0; i < text_def.fLines.arr.length; i++) {
                text_def.fLines.arr[i].fTextSize = text_size;
            }

            let text_def_ref = JSROOT.create("TPaveText");
            g_cur.fFunctions.Add(text_def_ref);
            Object.assign(text_def_ref, { fX1NDC: 0.3, fY1NDC: 0.6, fX2NDC: 0.9, fY2NDC: 0.7 });
            text_def_ref.AddText("Reference "+data["ref_id"]+" from "+data["ref_date"]+" run = "+data["ref_run_id"][0]);
            text_def_ref.fTextColor = 4;
            text_def_ref.fTextAlign = 11;
            for (let i = 0; i < text_def_ref.fLines.arr.length; i++) {
                text_def_ref.fLines.arr[i].fTextSize = text_size;
            }

            let mgraph = JSROOT.createTMultiGraph(g_cur, g_ref);

            let h1 = JSROOT.createHistogram('TH1I', npoints);
            h1.fName = 'axis_draw';
            h1.fTitle = null;
            h1.fXaxis = g_cur_tree.draw_object.fHistogram.fXaxis;
            h1.fYaxis = g_cur_tree.draw_object.fHistogram.fYaxis;
            h1.fXaxis.fTitle = "Time [ns]";
            h1.fYaxis.fTitle = "Amplitude";
            h1.fYaxis.fTitleOffset = 1.4;
            h1.fXaxis.fTitleOffset = 1.2;
            h1.fYaxis.fTitleFont = 62;
            h1.fXaxis.fTitleFont = 62;
            mgraph.fHistogram = h1;

            let mgraph_plot = await JSROOT.draw(plot_id, mgraph, "LP");
        }
        catch(err) {
            console.log(err);
            console.log("can't draw pulse shape");
        }
        shape_plot.setAttribute("name", "computed");
    }
}

I think the reason why I did not see anything plotted after requiring only first N entries is probably because none of the entries pass my selection. Now I wonder if there is some way how to stop the TTree drawing once I reach lets say 10k points in the resulting graph - meaning that I will not specify to process first 10k but process any number of events that will fill the graph with 10k values and then stop.

But still I find it much slower even when I specify only 1000 entries to the draw call for this large file as it would be if I have small root file with only 1000 entries. Is this expected?

Any suggestion how to further speed up the code are very welcome.

Thanks,
Dominik

5

Hi Dominik,

Can I get access to your files to reproduce problem?
You can send me link privately if you want.

For the moment there is no possibility to stop tree processing after getting N points in result graph object, but I can check if something can be done.

If you constructing TMultiGraph - I recommend to use treeDraw function which creates result object without making drawing on the canvas. Like in the example:

const file = await openFile('https://root.cern/js/files/hsimple.root');
const ntuple = await file.readObject('ntuple');
const graph = await treeDraw(ntuple, 'px:py::pz>5;graph');

One gets TGraph object directly.

Regards,
Sergey

6

Hi Dominik,

Normally JSROOT uses multi-range requests to get several chunks of TTree per single http request. But not all servers support such functionality. If such request fails - JSROOT try to switch to less number of ranges per request. Reducing finally to single range per request.

In such extreme situation latency of each request start play important role.
In your example there are about 500 different portions are requested - making together very long execution time.

You should check configuration of your server - that it really supports several ranges in one http request. https://httpd.apache.org/docs/2.4/en/mod/core.html#maxranges configuration for Apache server.

Regards,
Sergey

10

In JSROOT master branch I also add new parameter to treeDraw expression - nmatch:N.
After exactly N processed entries drawing will be stopped and result created.

In your particular case your expression selects exactly one entry.
Therefore you can add nmatch:1 to your parameters.

With some luck such entry placed in the begin of the file - and one get result much faster.

11

Hi Dominik,

I implement another alternative to significantly speed-up such TTree::Draw expression.
You are using several simple values for selector, but in the draw expression large array with 768 elements is included. When from 166380 entries only one single entry will be selected - more than 500MB will be loaded for nothing. If use expression directly:

let hist = await treeDraw(tree, 'Amplitude::barrel_ec == 0 && pos_neg == 0 && FT == 2 && slot == 4 && channel == 84 && corrUndo == 0;graph;');

with local server it takes 25 seconds. Plain ROOT requires 5 second.

If one sure that only single entry need to be matched, one can add nmatch:1 to the expression.
But JSROOT will try to load all data consequently - mean up to 25 seconds may be required.

Now I implement staged approach. Here one first search for all entries which are match cut condition and then perform drawing only for these entries. Only only require to add staged; argument to draw expression:

let hist = await treeDraw(tree, 'Amplitude::barrel_ec == 0 && pos_neg == 0 && FT == 2 && slot == 4 && channel == 84 && corrUndo == 0;graph;staged;');

And it takes 0.9 seconds on the same server - speedup of 25 times.

I guess it is worth to try for your usecase. Code is in JSROOT master branch.

Regards,
Sergey

12

Hi Sergey,

thanks a lot for all this additional features, they work really great :wink: . However, I am still not able to make my flask app work properly with the multi range requests. I am quite sure that this is not misconfiguration of the webserver as it works nicely when trying simple demo in plain html with JS. The single http request to the root file are of 13MB on average, but in my flask app is is still either several hundred B or max 15kB.

I tried to pass root file through specific route where I forced response headers to accept several ranges, but without any success, JSROOT always takes very small chunks of file. Configuration of my flask app is pretty standard and setting ‘MaxRanges unlimited’ does not have any effect on the performance or requests. Do you have experience with large root files accessed from the flask? I believe that these issues are specifically caused by some misconfiguration between flask and jsroot.

Best,
Dominik

13

How can I myself debug it?
JSROOT has complex logic to detect problems with multi-range requests and it may fail with flask server.

14

Hi Sergey,

I was able to reproduce the issue on the simplest possible flask app.
Here is the structure I used for testing:
virtual host config:

<VirtualHost *:80>
        ServerAdmin webmaster@localhost
        ServerName gci.exmpale.com
        DocumentRoot /var/www
        DirectoryIndex index.html index.php

        ErrorLog ${APACHE_LOG_DIR}/error.log
        CustomLog ${APACHE_LOG_DIR}/access.log combined
        MaxRanges 200

        <Directory /var/www/html_test>
                Options Indexes FollowSymLinks
                AllowOverride All
                Require all granted
        </Directory>

        WSGIScriptAlias /flask_test /var/www/flask_test/flask-app.wsgi
        WSGIDaemonProcess webTestflask python-path=/var/www/flask_test python-home=/var/www/flask_test/flask-venv
        <Directory /var/www/flask_test>
                WSGIProcessGroup webTestflask
                WSGIApplicationGroup %{GLOBAL}
                Order deny,allow
                Require all granted
                <Files app.py>
                        Require all granted
                </Files>
        </Directory>

</VirtualHost>

flask_test directory contains:
flask-app.wsgi

import sys
from os import environ, getcwd

from app import app as application

app/__init__.py

from flask import Flask, request, render_template, Response, send_file, send_from_directory
from pathlib import Path

print("hello")
app = Flask(__name__,
            template_folder=f"{Path(__file__).parent.parent.absolute()}/templates",
            static_folder=f"{Path(__file__).parent.parent.absolute()}/static" )
app.debug = True

@app.route("/", methods=["GET"])
def root_page():
    return render_template("index.html")

templates/index.html

<!DOCTYPE html>

<html lang="en">
   <body>
      <div id="drawing" style="width:600px; height:400px"></div>
   </body>

   <script type='module'>
      import { openFile, draw} from 'https://root.cern/js/latest/modules/main.mjs';
      let file = await openFile('./static/cur.root');
      let tree = await file.readObject('CALIWAVE');
      await draw("drawing", tree, 'Amplitude::barrel_ec == 0 && pos_neg == 0 && FT == 2 && slot == 4 && channel == 84 && corrUndo == 0;graph');
   </script>
</html>

Then there is flask-venv dir with python venv where the flask is installed and static subfolder with the root file cur.root.

the direcotry html_test contains just index.html as above. The webpage with html_test (which transfers root file in 13MB bunches) runs for about 40s while flask_test page (where the bunches are of 15kB size) for almost 4 mins.

Thanks,
Dominik

15

That is file name?
Where to place it?

And how to run Flask?

16

you can just add this new directory to the apache config as you would do for any other webpage. This is usually located in /etc/apache2/sites-enabled/ , so if you do not have any directories in the apache you can just put this whole virtual host config there (after installing apache of course) and call it as you wish, e.g. test.conf.

Simple instructions how to setup apache on linux - Install and Configure Apache | Ubuntu

Flask will be run automatically after sudo service apache2 restart, just go to http://127.0.0.1/flask_test/

17

You are using Apache to serve ROOT files?
And Flask server used for redirection to Apache server?

18

exactly, it is flask web application running on the apache webserver

19

Hi,

I install apache and Flask on my host,
but still have problem to run your application.

Before we continue this way - lets dry different approach.
In your index.html file I see:

let file = await openFile('./static/cur.root');

This means that you using static/ folder of Flask.
And I have doubts that Flask support multi-range requests at this place.
Can you put your files to other sub-directory of the Apache server?

Regards,
Sergey

20

And you always can check if your server support ranges using curl with arguments like:

curl -ik -L https://root.cern/js/files/hsimple.root -H "Range: bytes=0-15,1000-1015" --output -

You should get something like:

HTTP/1.1 206 Partial Content
date: Tue, 15 Apr 2025 07:24:49 GMT
server: Apache
last-modified: Wed, 08 Jan 2025 11:25:47 GMT
accept-ranges: bytes
content-length: 177
access-control-allow-origin: *
access-control-allow-headers: range
access-control-expose-headers: content-range,content-length,content-type,accept-ranges
access-control-allow-methods: HEAD,GET
content-type: multipart/byteranges; boundary=6c72110ec615eaaa
set-cookie: ee808f8747c47049fb3c8cee8456fada=682862081f00609f6ddc1994edc1481b; path=/; HttpOnly; Secure; SameSite=None
cache-control: private


--6c72110ec615eaaa
Content-range: bytes 0-15/414239

root�mdR
--6c72110ec615eaaa
Content-range: bytes 1000-1015/414239

1ZGqP��
--6c72110ec615eaaa--
21

Hi Sergey,

I believe the static files that are only read should be placed in the static folder. Once you define static folder in the __init__.py you can access static files. I tried your Range request test with curl and it seems like Range requests are indeed not supported by default in flask.

When I try to request even simple png file with Range I get 416 error:

curl -ik -L http://127.0.0.1/flask_test/static/files.png -H "Range: bytes=0-15,1000-1015" --output -

resulting in:

HTTP/1.1 416 REQUESTED RANGE NOT SATISFIABLE
Date: Tue, 15 Apr 2025 08:57:21 GMT
Server: Apache/2.4.58 (Ubuntu)
Content-Range: bytes */86020
Content-Length: 177
Content-Type: text/html; charset=utf-8

<!doctype html>
<html lang=en>
<title>416 Requested Range Not Satisfiable</title>
<h1>Requested Range Not Satisfiable</h1>
<p>The server cannot provide the requested range.</p>

That’s why I tried to manually pipe files (root, png) through /files route with explicitly allowing for Range requests, modyfying __init__.py file:

from flask import Flask, request, render_template, Response, send_file, send_from_directory
from pathlib import Path
import os, re

app = Flask(__name__,
            template_folder=f"{Path(__file__).parent.parent.absolute()}/templates",
            static_folder=f"{Path(__file__).parent.parent.absolute()}/static" )
app.debug = True

@app.route("/", methods=["GET"])
def root_page():
    return render_template("index.html")

def send_file_partial(file_path):
    """
    Serve a file with support for HTTP Range Requests.
    """
    range_header = request.headers.get('Range', None)
    if not range_header:
        # If no Range header is present, serve the entire file
        return send_file(file_path)

    # Get the file size
    file_size = os.path.getsize(file_path)

    # Parse the Range header
    byte1, byte2 = 0, None
    match = re.match(r"bytes=(\d+)-(\d*)", range_header)
    if match:
        byte1 = int(match.group(1))
        if match.group(2):
            byte2 = int(match.group(2))

    # Ensure byte2 is within the file size
    byte2 = byte2 if byte2 is not None else file_size - 1

    # Validate the range
    if byte1 >= file_size or byte2 >= file_size:
        return Response(status=416, headers={"Content-Range": f"bytes */{file_size}"})

    # Calculate the length of the requested range
    length = byte2 - byte1 + 1

    # Open the file and read the requested range
    with open(file_path, "rb") as f:
        f.seek(byte1)
        data = f.read(length)

    # Create the response with the requested range
    response = Response(data, status=206, mimetype="application/octet-stream")
    response.headers.add("Content-Range", f"bytes {byte1}-{byte2}/{file_size}")
    response.headers.add("Accept-Ranges", "bytes")
    response.headers.add("Content-Length", str(length))
    return response


@app.route('/files/<path:filename>')
def serve_file(filename):
    """
    Serve .root files with range support.
    """
    file_path = os.path.join(app.static_folder, filename)  # Adjust the path to your files
    if not os.path.exists(file_path):
        return Response("File not found", status=404)
    return send_file_partial(file_path)

However this removes 416 error and now allow for range requests, it still does not work properly with JSROOT. Can this be caused by JSROOT which still evaluates the request as non-multirange and breaks it to small pieces in the request? Now the range request works fine but looks like JSROOT asks for just single small ranges…

Thanks,
Dominik

22

Hi Dominik,

If I am right - send_file_partial function only able to handle single range at once.
Multi-part ranges requests are much more complicated:

First of all, in header one should have:

content-type: multipart/byteranges; boundary=6c72110ec615eaaa

And data split on portions - each separated with boundary string.
See my example. And documentation on mozilla.org

Regards,
Sergey

23

Yes, I know this is more complicated, I tried to accomodate for this and modified the send_file_partial function as follows:

def send_file_partial(file_path):
    """
    Serve a file with support for HTTP Range Requests, including multi-range requests.
    """
    range_header = request.headers.get('Range', None)
    file_size = os.path.getsize(file_path)

    if not range_header:
        # If no Range header is present, serve the entire file
        return send_file(file_path)

    # Parse the Range header
    ranges = []
    match = re.finditer(r"bytes=(\d+)-(\d*)", range_header)
    for m in match:
        byte1 = int(m.group(1))
        byte2 = int(m.group(2)) if m.group(2) else file_size - 1
        if byte1 >= file_size or byte2 >= file_size or byte1 > byte2:
            return Response(status=416, headers={"Content-Range": f"bytes */{file_size}"})
        ranges.append((byte1, byte2))

    # Determine the MIME type of the file
    mime_type, _ = mimetypes.guess_type(file_path)
    mime_type = mime_type or "application/octet-stream"

    if len(ranges) == 1:
        # Single range request
        byte1, byte2 = ranges[0]
        length = byte2 - byte1 + 1

        # Stream the single range
        def generate_single_range(file_path, start, length):
            with open(file_path, "rb") as f:
                f.seek(start)
                remaining = length
                while remaining > 0:
                    chunk_size = min(8192, remaining)  # Read in 8 KB chunks
                    data = f.read(chunk_size)
                    if not data:
                        break
                    yield data
                    remaining -= len(data)

        response = Response(generate_single_range(file_path, byte1, length), status=206, mimetype=mime_type)
        response.headers.add("Content-Range", f"bytes {byte1}-{byte2}/{file_size}")
        response.headers.add("Accept-Ranges", "bytes")
        response.headers.add("Content-Length", str(length))
        return response

    # Multi-range request
    boundary = "MULTIPART_BYTERANGES_BOUNDARY"

    def generate_multi_range(file_path, ranges):
        with open(file_path, "rb") as f:
            for byte1, byte2 in ranges:
                f.seek(byte1)
                length = byte2 - byte1 + 1
                data = f.read(length)

                # Yield the boundary and headers for this range
                yield f"--{boundary}\r\n"
                yield f"Content-Type: {mime_type}\r\n"
                yield f"Content-Range: bytes {byte1}-{byte2}/{file_size}\r\n\r\n"

                # Yield the actual data
                yield data
                yield b"\r\n"

            # End the multipart response
            yield f"--{boundary}--\r\n"

    response = Response(generate_multi_range(file_path, ranges), status=206, mimetype=f"multipart/byteranges; boundary={boundary}")
    response.headers.add("Accept-Ranges", "bytes")
    return response

This should now use the multirange request if there are multiple request ranges, but Range headers in requests are still single, like this:

image
image541×233 20.9 KB

That’s why I suspect that JSROOT by some logic always slips to single range requests for flask… Is there some way how to force jsroot to send multirange request in any case?