IMPALA-12364: Display memory, disk and network metrics in webUI's query timeline

This patch adds fragment-level metrics to the WebUI query timeline
display along with additional disk and network metrics.

The fragment's plan nodes are enlarged with an animated transition on
hovering over the fragment's row in query timeline's fragment diagram.

On clicking the plan nodes, total thread and memory usage of the parent
fragment are displayed, after accumulating memory and thread usage of
all child nodes. Thread usage is being shown on the additional Y-axis.

In this way, memory and thread usage of multiple fragments can be
compared alongside. A fragment's usage can be hidden by clicking
on any of the child plan nodes again.

These counters are available within the profile with following names.

- MemoryUsage
- ThreadUsage

Once a fragment's metrics are displayed, they are updated as they
are collected from the profile during a running query.

A grid-line is displayed along with a tooltip on hovering over the
fragment diagram, containing the instantaneous time at that position.
This grid-line also triggers tooltips and gridlines in other charts.

A warning is displayed on clicking a fragment with less number of samples
available.

RESOURCE_TRACE_RATIO query option must be set for providing periodic
metrics within the profile. This allows the following time series
counters to be displayed on the query timeline.

- HostDiskWriteThroughput
- HostDiskReadThroughput
- HostNetworkRx
- HostNetworkTx

The additional Y-axis within the utilization chart is used to represent
the average of these metrics.

The memory units in tooltips and ticks on co-ordinate axes are displayed
in human readable form such as KB, MB, GB and PB for convenience.

Both of the charts contain controls to close the chart. These charts
can also be resized until a maximum and minmum limit by dragging the
resize bar's handle.

Along with mouse wheel events, the diagrams can be horizontally
stretched by the help of buttons with horizontal zoom icons at the
top of the page. The zoom out button is disabled, when further zoom out
is not possible.

Timeticks are being autoscaled during fragment diagram's horizontal zoom.

In addition to the scrollbar, hovering on edges of the window allows
horizontal scrolling.

Test cases have been for the additional disk, network and fragment level
memory metrics parsing functions.

Change-Id: Ifd25e6f0bc9fbd664ec98936daff3f27182dfc7f
Reviewed-on: http://gerrit.cloudera.org:8080/20355
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

bin/rat_exclude_files.txt

@@ -44,6 +44,7 @@ www/bootstrap/css/bootstrap*
 www/bootstrap/js/bootstrap*
 www/favicon.ico
 www/Chart*
+www/icons/*
 tests/comparison/leopard/static/css/bootstrap*
 tests/comparison/leopard/static/fonts/glyphicons-halflings*
 tests/comparison/leopard/static/js/bootstrap*

www/query_timeline.tmpl

@@ -35,6 +35,45 @@ under the License.
 }
 </style>
 
+<style>
+g.plan_node:hover {
+  transform-box: fill-box;
+  transform: translateY(-50%) scaleY(1.50);
+}
+
+g.plan_node:active {
+  transform-box: fill-box;
+  transform: translateY(-100%) scaleY(2);
+}
+
+#host_utilization_wrapper, #fragment_metrics_wrapper {
+  display: none;
+  user-select: none;
+}
+
+#host_utilization_resize_bar, #fragment_metrics_resize_bar {
+  display: inline-block;
+  cursor: row-resize;
+  user-select: none;
+  height: 6px;
+}
+
+#hor_zoomout {
+  background-image: url("www/icons/horizontal_zoomout.png");
+}
+
+#hor_zoomin {
+  background-image: url("www/icons/horizontal_zoomin.png");
+}
+
+#hor_zoomout, #hor_zoomin{
+  background-size: cover;
+  height: 1.75em;
+  width: 1.75em;
+  cursor: pointer;
+}
+</style>
+
 {{> www/query_detail_tabs.tmpl }}
 
 
@@ -45,6 +84,10 @@ under the License.
 {{^plan_metadata_unavailable}}
   <div style="display:flex; justify-content:space-between;">
     <h3>Timeline</h3>
+    <span id="controls">
+      <button id="hor_zoomin" class="btn btn-primary"></button>
+      <button id="hor_zoomout" class="btn btn-primary" disabled></button>
+    </span>
     <label>
       <h4 style="display:inline;"> Download : </h4>
       <input type="button" class="btn btn-primary" data-toggle="modal" value="HTML"
@@ -93,9 +136,21 @@ under the License.
     <svg id="timeticks_footer"></svg>
   </div>
 </div>
-<div id="host_utilization_diagram">
+<div id="host_utilization_wrapper">
-  <!--Utilization metrics is not available. Please make sure to set query option
+  <img id="host_utilization_resize_bar" src="www/icons/drag_handle.png" draggable="false"/>
-    RESOURCE_TRACE_RATIO=true.-->
+  <input id="host_utilization_close_btn" class="btn btn-primary btn-sm" type="button"
+      value="X" data-dismiss="modal"/>
+  <div id="host_utilization_diagram">
+    <!--Utilization metrics is not available. Please make sure to set query option
+      RESOURCE_TRACE_RATIO=true.-->
+  </div>
+</div>
+<div id="fragment_metrics_wrapper">
+  <img id="fragment_metrics_resize_bar" src="www/icons/drag_handle.png" draggable="false"/>
+  <input id="fragment_metrics_close_btn" class="btn btn-primary btn-sm" type="button"
+      value="X" data-dismiss="modal"/>
+  <div id="fragment_metrics_diagram">
+  </div>
 </div>
 
 {{/plan_metadata_unavailable}}
@@ -109,6 +164,8 @@ import {renderTimingDiagram, collectFragmentEventsFromProfile, ntics, set_ntics}
     "./www/scripts/query_timeline/fragment_diagram.js";
 import {collectUtilizationFromProfile, toogleUtilizationVisibility} from
     "/www/scripts/query_timeline/host_utilization_diagram.js";
+import {collectFragmentMetricsFromProfile} from
+    "/www/scripts/query_timeline/fragment_metrics_diagram.js";
 import {profile, set_profile, diagram_width, set_diagram_width, border_stroke_width,
     resizeHorizontalAll, maxts} from "/www/scripts/query_timeline/global_members.js";
 
@@ -122,6 +179,7 @@ window.onload = function refresh() {
       set_profile(JSON.parse(req.responseText)["profile_json"]);
       collectFragmentEventsFromProfile();
       collectUtilizationFromProfile();
+      collectFragmentMetricsFromProfile();
       if (last_maxts != maxts) {
         renderTimingDiagram();
         last_maxts = maxts;
@@ -137,11 +195,49 @@ window.onload = function refresh() {
   req.send();
 }
 
+window.addEventListener('mousemove', function(e) {
+  var scroll_margin = window.innerWidth / 10;
+  var scroll_amount = window.innerWidth / 3;
+  if (e.clientX <= scroll_margin) {
+    window.scrollBy({left : - scroll_amount, behavior : "smooth"});
+  } else if (e.clientX >= window.innerWidth - scroll_margin) {
+    window.scrollBy({left : + scroll_amount, behavior : "smooth"});
+  }
+});
+
 window.addEventListener('resize', function(event) {
-  set_diagram_width(Math.max(window.innerWidth, diagram_width - border_stroke_width));
+  var window_diagram_width = window.innerWidth - border_stroke_width;
+  if (diagram_width <= window_diagram_width) {
+    set_diagram_width(window_diagram_width);
+    hor_zoomout.disabled = true;
+  } else {
+    hor_zoomout.disabled = false;
+  }
   resizeHorizontalAll();
 }, true);
 
+hor_zoomin.addEventListener('click', function(event) {
+  set_diagram_width(diagram_width + window.innerWidth * 0.2)
+  set_ntics(ntics + 2);
+  resizeHorizontalAll();
+  hor_zoomout.disabled = false;
+});
+
+hor_zoomout.addEventListener('click', function(event) {
+  var prev_diagram_width = diagram_width;
+  var next_width = diagram_width - window.innerWidth * 0.2;
+  var window_diagram_width = window.innerWidth - border_stroke_width;
+  if (next_width <= window_diagram_width) {
+    next_width = window_diagram_width;
+    hor_zoomout.disabled = true;
+  }
+  set_diagram_width(next_width);
+  var next_ntics = ntics + (diagram_width - prev_diagram_width) * 10 / window.innerWidth;
+  next_ntics = Math.round(next_ntics);
+  set_ntics(next_ntics);
+  resizeHorizontalAll();
+});
+
 // Attaches a SVG blob of the complete timeline to the associated link
 export_link.addEventListener('click', function (event) {
   if (export_format.value == ".html") {

www/scripts/query_timeline/chart_commons.js

@@ -19,6 +19,11 @@ import {maxts, set_maxts, clearDOMChildren} from "./global_members.js";
 import {name_width} from "./fragment_diagram.js";
 
 export var exportedForTest;
+// In the data array provided for c3's line chart,
+// 0th element is name of dataset
+// 1st element is zero in all datasets(to compensate missing value for line chart)
+// Hence the values index start at 2
+export var array_values_start_index = 2;
 
 function accumulateTimeseriesValues(values_array, time_series_counter, max_samples) {
   var samples = time_series_counter.data.split(",").map(el => parseInt(el));
@@ -77,7 +82,7 @@ export function mapTimeseriesCounters(time_series_counters, counters) {
 
 export function clearTimeseriesValues(values_array, max_samples) {
   var max_traverse_len = Math.min(max_samples.available + 1, values_array.length - 1);
-  for (var j = 2; j <= max_traverse_len; ++j) {
+  for (var j = array_values_start_index; j <= max_traverse_len; ++j) {
     values_array[j] = null;
   }
 }
@@ -93,6 +98,17 @@ export function aggregateProfileTimeseries(parent_profile, aggregate_array,
   });
 }
 
+export function showTooltip(chart, x) {
+  if (chart == undefined) return;
+  chart.tooltip.show({x : chart.internal.findClosestFromTargetsByX(
+      chart.internal.getTargets(), x).x});
+}
+
+export function hideTooltip(chart) {
+  if (chart == undefined) return;
+  chart.tooltip.hide();
+}
+
 export function displayWarning(parent_element, msg, diagram_width, margin_l, margin_r) {
   clearDOMChildren(parent_element);
   parent_element.style.display = "flex";

www/scripts/query_timeline/fragment_diagram.js

@@ -19,8 +19,11 @@ import {profile, set_maxts, maxts, decimals, set_decimals, diagram_width,
     set_diagram_width, diagram_controls_height, diagram_min_height,
     margin_header_footer, border_stroke_width, margin_chart_end, clearDOMChildren,
     resizeHorizontalAll} from "./global_members.js";
-import {host_utilization_chart, getUtilizationHeight} from
+import {host_utilization_chart, getUtilizationWrapperHeight}
-    "./host_utilization_diagram.js";
+    from "./host_utilization_diagram.js";
+import {fragment_metrics_chart, getFragmentMetricsWrapperHeight,
+    updateFragmentMetricsChartOnClick} from "./fragment_metrics_diagram.js";
+import {showTooltip, hideTooltip} from "./chart_commons.js"
 import "./global_dom.js";
 
 export var exportedForTest;
@@ -56,6 +59,7 @@ var receiver_nodes = [];
 var max_namelen = 0;
 var frag_name_width;
 var chart_width;
+var timestamp_gridline;
 
 // #timeticks_footer
 export var ntics = 10;
@@ -201,14 +205,15 @@ async function renderFragmentDiagram() {
           row_height, false));
 
       // Fragment/sender timing row
-      DrawBars(fragment_diagram, rownum_l, row_height, fevents, name_width, px_per_ns);
+      var fragment_svg_group = getSvgGroup();
+      DrawBars(fragment_svg_group, rownum_l, row_height, fevents, name_width, px_per_ns);
 
       for (var i = 0; i < fragment.nodes.length; ++i) {
         var node = fragment.nodes[i];
 
         if (node.events != undefined) {
           // Plan node timing row
-          DrawBars(fragment_diagram, rownum_l, row_height, node.events, name_width,
+          DrawBars(fragment_svg_group, rownum_l, row_height, node.events, name_width,
               px_per_ns);
           if (node.type == "HASH_JOIN_NODE") {
             fragment_diagram.appendChild(getSvgText("X", stroke_fill_colors.black,
@@ -284,6 +289,9 @@ async function renderFragmentDiagram() {
         }
 
       }
+      fragment_svg_group.id = fragment.name;
+      fragment_svg_group.addEventListener('click', updateFragmentMetricsChartOnClick);
+      fragment_diagram.appendChild(fragment_svg_group);
 
       // Visit sender fragments in reverse order to avoid dag edges crossing
       pending_fragments.reverse().forEach(printFragment);
@@ -527,7 +535,8 @@ export function setTimingDiagramDimensions(ignored_arg) {
   page_additional_height = timing_diagram.offsetTop + (row_height + margin_header_footer
       + border_stroke_width * 4) * 2;
   var fragment_diagram_initial_height = window.innerHeight - page_additional_height;
-  var remaining_height = fragment_diagram_initial_height - getUtilizationHeight();
+  var remaining_height = fragment_diagram_initial_height - getUtilizationWrapperHeight()
+      - getFragmentMetricsWrapperHeight();
   var display_height = Math.max(diagram_min_height, Math.min(remaining_height,
       rownum * row_height));
   chart_width = diagram_width - name_width - margin_chart_end - border_stroke_width;
@@ -557,17 +566,47 @@ export function renderTimingDiagram() {
   }
 }
 
+fragment_diagram.addEventListener('mouseout', function(e) {
+  hideTooltip(host_utilization_chart);
+  hideTooltip(fragment_metrics_chart);
+  removeChildIfExists(fragment_diagram, timestamp_gridline);
+});
+
+fragment_diagram.addEventListener('mousemove', function(e) {
+  if (e.pageX >= name_width && e.pageX <= name_width + chart_width){
+    removeChildIfExists(fragment_diagram, timestamp_gridline);
+    timestamp_gridline = getSvgLine(stroke_fill_colors.black, e.pageX, 0, e.pageX,
+        parseInt(fragment_diagram.style.height));
+    fragment_diagram.appendChild(timestamp_gridline);
+    var gridline_time = ((maxts * (e.pageX - name_width) / chart_width) / 1e9);
+    showTooltip(host_utilization_chart, gridline_time);
+    showTooltip(fragment_metrics_chart, gridline_time);
+    fragment_diagram_title.textContent = gridline_time.toFixed(decimals) + " s";
+  } else {
+    try {
+      host_utilization_chart.tooltip.hide();
+    } catch (e) {
+    }
+    removeChildIfExists(fragment_diagram, timestamp_gridline);
+    fragment_diagram_title.textContent = "";
+  }
+});
+
 fragment_diagram.addEventListener('wheel', function(e) {
   if (e.shiftKey) {
-    if (e.wheelDelta <= 0 && diagram_width <= window.innerWidth) return;
+    var window_diagram_width = window.innerWidth - border_stroke_width;
+    if (e.wheelDelta <= 0 && diagram_width <= window_diagram_width) return;
     var next_diagram_width = diagram_width + Math.round(e.wheelDelta);
-    if (next_diagram_width <= window.innerWidth) {
+    hor_zoomout.disabled = false;
-      next_diagram_width = window.innerWidth;
+    if (next_diagram_width <= window_diagram_width) {
+      next_diagram_width = window_diagram_width;
+      hor_zoomout.disabled = true;
     }
     var next_chart_width = next_diagram_width - name_width - margin_chart_end
         - border_stroke_width;
     var next_ntics = (next_diagram_width - diagram_width) * 10 / window.innerWidth;
     next_ntics = ntics + Math.round(next_ntics);
+    if (next_ntics < 10) next_ntics = 10;
     var rendering_constraint = char_width * (decimals + integer_part_estimate) >=
         next_chart_width / next_ntics;
     if (rendering_constraint) return;

www/scripts/query_timeline/fragment_metrics_diagram.js

@@ -0,0 +1,309 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import {name_width, page_additional_height, setTimingDiagramDimensions}
+    from "./fragment_diagram.js"
+import {profile, diagram_width, diagram_controls_height, diagram_min_height,
+    resizeVerticalAll, margin_chart_end, decimals} from "./global_members.js";
+import {aggregateProfileTimeseries, generateTimesamples, clearTimeseriesValues,
+    mapTimeseriesCounters, displayWarning, destroyChart} from "./chart_commons.js";
+import {resetUtilizationHeight, resizeUtilizationChart, getUtilizationWrapperHeight}
+    from "./host_utilization_diagram.js";
+import "./global_dom.js";
+
+export var exportedForTest;
+
+export var fragment_id_selections = new Map();
+export var fragment_metrics_parse_successful = false;
+export var fragment_metrics_chart = null;
+
+var fragment_metrics_aggregate;
+var fragment_metrics_counters = [
+  ["MemoryUsage", "memory usage", 0],
+  ["ThreadUsage", "thread usage", 0]
+];
+var sampled_fragment_metrics_timeseries;
+var fragment_metrics_timeaxis_name = "fragment metrics timeticks";
+var max_samples_fragment_metrics = {
+  allocated : 1000,
+  available : 0,
+  collected : 0,
+  period : null
+};
+var fragment_metrics_resize_factor = 0.2;
+
+var fragment_metrics_close_btn = document.getElementById("fragment_metrics_close_btn");
+var host_utilization_resize_bar = document.getElementById("fragment_metrics_resize_bar");
+
+function initializeFragmentMetricsChart() {
+  fragment_metrics_diagram.style = null;
+  fragment_metrics_chart = c3.generate({
+    bindto : "#fragment_metrics_diagram",
+    data : {
+      columns : [[fragment_metrics_timeaxis_name, 0]],
+      type : "area",
+      order : "asc",
+      x : fragment_metrics_timeaxis_name
+    }, size : {
+      height : getFragmentMetricsHeight(),
+      width : diagram_width
+    }, padding : {
+      left : name_width,
+      right : margin_chart_end
+    }, axis : {
+      x :
+      {
+        padding : {
+          left : 0,
+          right : 0
+        },
+        tick : {
+          format : function (x) { return x.toFixed(decimals); }
+        }
+      },
+      y :
+      {
+        tick : {
+          format : function (y) { return getReadableSize(y, 1); }
+        }
+      },
+      y2 :
+      {
+        tick : {
+          format : function (y2) { return (y2 == Math.floor(y2) ? y2 : ""); }
+        },
+        show : true
+      }
+    }, legend : {
+      show : false
+    }, tooltip : {
+      format : {
+        value : function (value, ratio, id, index) {
+          if (id.includes("memory usage")){
+            return getReadableSize(value, decimals);
+          } else {
+            return value + " Thread(s)";
+          }
+        },
+        title : function (x, index) {
+          return x.toFixed(decimals) + "s";
+        }
+      }
+    }
+  });
+  fragment_metrics_chart.load({
+    unload : true
+  });
+  var chart_width = diagram_width - margin_chart_end - name_width;
+  fragment_metrics_resize_bar.style.marginLeft = `${name_width + chart_width / 4}px`;
+  fragment_metrics_resize_bar.style.width = `${chart_width / 2}px`;
+  fragment_metrics_resize_bar.style.marginRight = `${chart_width / 4}px`;
+}
+
+function dragResizeBar(mousemove_e) {
+  if (mousemove_e.target.classList[0] == "c3-event-rect") return;
+  var next_height = getFragmentMetricsHeight() + (fragment_metrics_resize_bar.offsetTop -
+      mousemove_e.clientY);
+  if (next_height >= diagram_min_height &&
+      window.innerHeight - next_height - diagram_controls_height >=
+      page_additional_height + getUtilizationWrapperHeight() + diagram_min_height) {
+    fragment_metrics_resize_factor = next_height / window.innerHeight;
+  }
+  resizeVerticalAll();
+}
+
+function initializeFragmentMetrics(parent_profile, counters, max_samples, timeaxis_name) {
+  // user, sys, io and sampled timeticks
+  max_samples.available = 0;
+  max_samples.period = 0;
+  var fragment_metrics_aggregate = new Array(counters.length);
+  for (var i = 0; i < counters.length; ++i) {
+    fragment_metrics_aggregate[i] = new Array(max_samples.allocated + 3)
+        .fill(null);
+    fragment_metrics_aggregate[i][1] = 0;
+  }
+  var sampled_fragment_metrics_timeseries = new Array(max_samples.allocated + 3)
+      .fill(null);
+  sampled_fragment_metrics_timeseries[0] = timeaxis_name;
+  mapTimeseriesCounters(parent_profile.child_profiles[0].time_series_counters, counters);
+  return {fragment_metrics_aggregate, sampled_fragment_metrics_timeseries};
+}
+
+function getFragmentMetricsHeight() {
+  return Math.max(diagram_min_height, window.innerHeight *
+      fragment_metrics_resize_factor);
+}
+
+export function getFragmentMetricsWrapperHeight() {
+  return (fragment_metrics_parse_successful && fragment_id_selections.size > 0)
+      * (getFragmentMetricsHeight() + diagram_controls_height);
+}
+
+export function toogleFragmentMetricsVisibility() {
+  if (fragment_id_selections.size > 0 && fragment_metrics_parse_successful) {
+    fragment_metrics_wrapper.style.display = "initial";
+  } else {
+    fragment_metrics_wrapper.style.display = "none";
+  }
+}
+
+export function collectFragmentMetricsFromProfile() {
+  // do not collect metrics, in case a fragment is not selected
+  if (fragment_id_selections.size == 0) {
+    fragment_metrics_parse_successful = false;
+    return;
+  }
+  // try parsing the metrics from JSON, to seek case of errors in expected format
+  // and do not render the chart unless attributes are parsed accurately
+  try {
+    // "Execution Profile" contains the different fragment's counters and
+    // there averaged metrics
+    var profile_fragments = profile.child_profiles[2].child_profiles;
+
+    // for all the fragments selected in the execution profile
+    profile_fragments.every(function (fragment_profile) {
+      // initialize the collection arrays for subsequent collections
+      // arrays are overwritten without further re-allocation to reduce memory usage
+      if(fragment_id_selections.has(fragment_profile.profile_name)) {
+        if (fragment_metrics_chart == null) {
+          ({fragment_metrics_aggregate, sampled_fragment_metrics_timeseries} =
+              initializeFragmentMetrics(fragment_profile, fragment_metrics_counters,
+                  max_samples_fragment_metrics, fragment_metrics_timeaxis_name));
+          initializeFragmentMetricsChart();
+        }
+
+        // map counters to axes
+        var axes_mappings = {};
+        for (var i = 0; i < fragment_metrics_counters.length; ++i) {
+          fragment_metrics_aggregate[i][0] =
+              `${fragment_profile.profile_name} - ${fragment_metrics_counters[i][1]}`;
+        }
+        axes_mappings[fragment_metrics_aggregate[0][0]] = "y";
+        axes_mappings[fragment_metrics_aggregate[1][0]] = "y2";
+
+        // aggregate values from nodes of all fragment's plan nodes
+        aggregateProfileTimeseries(fragment_profile, fragment_metrics_aggregate,
+            fragment_metrics_counters, max_samples_fragment_metrics);
+
+        // display warnings in case less samples are available without plotting
+        if (max_samples_fragment_metrics.available <= 0) {
+          var fragment_metrics_samples_message =
+              `Warning: Not enough samples for fragment metrics plot.`;
+          fragment_metrics_chart = destroyChart(fragment_metrics_chart,
+              fragment_metrics_diagram);
+          displayWarning(fragment_metrics_diagram, fragment_metrics_samples_message,
+              diagram_width, name_width, margin_chart_end);
+          fragment_metrics_parse_successful = true;
+          return true;
+        }
+
+        // generate timestamps for utilization values and decide on last timestamp value
+        generateTimesamples(sampled_fragment_metrics_timeseries,
+            max_samples_fragment_metrics, true);
+
+        // load fragment metrics to the chart
+        fragment_metrics_chart.load({
+          columns : [...fragment_metrics_aggregate, sampled_fragment_metrics_timeseries],
+          axes : axes_mappings
+        });
+        // clear utilization value and timestamp samples arrays
+        fragment_metrics_aggregate.forEach(function (acc_usage) {
+          clearTimeseriesValues(acc_usage, max_samples_fragment_metrics);
+        });
+      }
+      return true;
+    });
+    fragment_metrics_parse_successful = true;
+  } catch (e) {
+    fragment_metrics_parse_successful = false;
+    fragment_metrics_chart = destroyChart(fragment_metrics_chart,
+        fragment_metrics_diagram);
+    console.log(e);
+  }
+}
+
+export async function resizeFragmentMetricsChart() {
+  if (fragment_metrics_chart == null) return;
+  var chart_width = diagram_width - margin_chart_end - name_width;
+  fragment_metrics_resize_bar.style.marginLeft = `${name_width + chart_width / 4}px`;
+  fragment_metrics_resize_bar.style.width = `${chart_width / 2}px`;
+  fragment_metrics_resize_bar.style.marginRight = `${chart_width / 4}px`;
+  fragment_metrics_chart.resize({
+    height : getFragmentMetricsHeight(),
+    width : diagram_width
+  });
+}
+
+export function updateFragmentMetricsChartOnClick(click_event) {
+  var selected_fragment_id = click_event.target.parentElement.parentElement.id;
+  fragment_id_selections.delete(selected_fragment_id);
+  if (fragment_metrics_chart != null) {
+    var unloaded = fragment_metrics_chart.internal.getTargets().every(function (target) {
+      if (target.id.includes(selected_fragment_id)) {
+        var unload_ids = new Array(fragment_metrics_counters.length);
+        for (var i = 0; i < fragment_metrics_counters.length; i++) {
+          unload_ids[i] =
+              `${selected_fragment_id} - ${fragment_metrics_counters[i][1]}`;
+        }
+        fragment_metrics_chart.unload({ids : unload_ids});
+        return false;
+      }
+      return true;
+    });
+    if (fragment_id_selections.size == 0) {
+      fragment_metrics_chart = destroyChart(fragment_metrics_chart,
+          fragment_metrics_diagram);
+      toogleFragmentMetricsVisibility();
+      setTimingDiagramDimensions();
+      return;
+    }
+    if (!unloaded) return;
+  }
+  if(fragment_id_selections.size == 0) {
+    resetUtilizationHeight();
+  }
+  fragment_id_selections.set(selected_fragment_id, 1);
+  collectFragmentMetricsFromProfile();
+  resizeUtilizationChart();
+  setTimingDiagramDimensions();
+  toogleFragmentMetricsVisibility();
+}
+
+fragment_metrics_resize_bar.addEventListener('mousedown',
+    function dragResizeBarBegin(mousedown_e) {
+  fragment_metrics_resize_bar.removeEventListener('mousedown', dragResizeBarBegin);
+  document.body.addEventListener('mousemove', dragResizeBar);
+  document.body.addEventListener('mouseup', function dragResrizeBarEnd() {
+    document.body.removeEventListener('mouseup', dragResrizeBarEnd);
+    document.body.removeEventListener('mousemove', dragResizeBar);
+    fragment_metrics_resize_bar.addEventListener('mousedown', dragResizeBarBegin);
+  });
+});
+
+fragment_metrics_close_btn.addEventListener('click', function() {
+  fragment_id_selections.clear();
+  fragment_metrics_chart = destroyChart(fragment_metrics_chart, fragment_metrics_diagram);
+  toogleFragmentMetricsVisibility();
+  setTimingDiagramDimensions();
+});
+
+fragment_metrics_close_btn.style.height = `${diagram_controls_height}px`;
+fragment_metrics_close_btn.style.fontSize = `${diagram_controls_height / 2}px`;
+
+if (typeof process != "undefined" && process.env.NODE_ENV === 'test') {
+  exportedForTest = {initializeFragmentMetrics};
+}

www/scripts/query_timeline/global_members.js

@@ -16,6 +16,7 @@
 // under the License.
 
 import {renderTimingDiagram, setTimingDiagramDimensions} from "./fragment_diagram.js";
+import {resizeFragmentMetricsChart} from "./fragment_metrics_diagram.js";
 import {resizeUtilizationChart} from "./host_utilization_diagram.js";
 
 export var profile = {};
@@ -53,9 +54,11 @@ export function clearDOMChildren(element) {
 export function resizeHorizontalAll() {
   renderTimingDiagram();
   resizeUtilizationChart();
+  resizeFragmentMetricsChart();
 }
 
 export function resizeVerticalAll() {
   setTimingDiagramDimensions();
   resizeUtilizationChart();
+  resizeFragmentMetricsChart();
 }

www/scripts/query_timeline/host_utilization_diagram.js

@@ -20,8 +20,10 @@ import {profile, set_maxts, maxts, diagram_width, diagram_controls_height,
     from "./global_members.js";
 import {name_width, page_additional_height, setTimingDiagramDimensions}
     from "./fragment_diagram.js";
-import {aggregateProfileTimeseries, generateTimesamples, clearTimeseriesValues,
+import {array_values_start_index, aggregateProfileTimeseries, generateTimesamples,
-    mapTimeseriesCounters, displayWarning, destroyChart} from "./chart_commons.js";
+    clearTimeseriesValues, mapTimeseriesCounters, displayWarning, destroyChart}
+    from "./chart_commons.js";
+import {getFragmentMetricsWrapperHeight} from "./fragment_metrics_diagram.js";
 import "./global_dom.js";
 
 export var exportedForTest;
@@ -36,7 +38,14 @@ var cpu_utilization_counters = [
     ["HostCpuSysPercentage", "avg sys", 0],
     ["HostCpuUserPercentage", "avg user", 0]
 ];
+var read_write_metrics_counters = [
+    ["HostDiskReadThroughput", "avg disk read", 0],
+    ["HostDiskWriteThroughput", "avg disk write", 0],
+    ["HostNetworkRx", "avg network rx", 0],
+    ["HostNetworkTx", "avg network tx", 0]
+];
 var cpu_nodes_usage_aggregate;
+var read_write_metrics_aggregate;
 var sampled_utilization_timeseries;
 var utilization_timeaxis_name = "utilization timeticks";
 var prev_utilization_num_samples = 0;
@@ -47,12 +56,19 @@ var max_samples_utilization = {
   period : null
 };
 var host_utilization_resize_factor = 0.2;
+var min_num_samples = 3;
+
+var host_utilization_close_btn = document.getElementById("host_utilization_close_btn");
+var host_utilization_resize_bar = document.getElementById("host_utilization_resize_bar");
 
 function initializeUtilizationChart() {
   var axis_mappings = {};
   for(var i = 0; i < cpu_utilization_counters.length; ++i) {
     axis_mappings[cpu_utilization_counters[i][1]] = "y";
   }
+  for(var i = 0; i < read_write_metrics_counters.length; ++i) {
+    axis_mappings[read_write_metrics_counters[i][1]] = "y2";
+  }
   var cpu_utilization_counter_group = new Array(cpu_utilization_counters.length);
   for (var i = 0; i < cpu_utilization_counters.length; i++){
     cpu_utilization_counter_group[i] = cpu_utilization_counters[i][1];
@@ -89,6 +105,13 @@ function initializeUtilizationChart() {
         tick : {
           format : function (y) { return y + '%'; }
         }
+      },
+      y2 :
+      {
+        tick : {
+          format : function (y2) { return `${getReadableSize(y2, 1)}/s`; }
+        },
+        show : true
       }
     }, legend : {
       show : false
@@ -111,10 +134,25 @@ function initializeUtilizationChart() {
     unload : true
   });
   var chart_width = diagram_width - margin_chart_end - name_width;
+  host_utilization_resize_bar.style.marginLeft = `${name_width + chart_width / 4}px`;
+  host_utilization_resize_bar.style.width = `${chart_width / 2}px`;
+  host_utilization_resize_bar.style.marginRight = `${chart_width / 4}px`;
 }
 
-function initializeUtilizationMetrics(parent_profile, counters_y1, max_samples,
+function dragResizeBar(mousemove_e) {
-    timeaxis_name) {
+  if (mousemove_e.target.classList[0] == "c3-event-rect") return;
+  var next_height = getUtilizationHeight() + (host_utilization_resize_bar.offsetTop -
+      mousemove_e.clientY);
+  if (next_height >= diagram_min_height && window.innerHeight - next_height
+      - diagram_controls_height >= page_additional_height
+      + getFragmentMetricsWrapperHeight() + diagram_min_height) {
+    host_utilization_resize_factor = next_height / window.innerHeight;
+    resizeVerticalAll();
+  }
+}
+
+function initializeUtilizationMetrics(parent_profile, counters_y1, counters_y2,
+    max_samples, timeaxis_name) {
   console.assert(parent_profile.profile_name == "Per Node Profiles");
   // user, sys, io and sampled timeticks
   var cpu_nodes_usage_aggregate = new Array(counters_y1.length);
@@ -126,29 +164,50 @@ function initializeUtilizationMetrics(parent_profile, counters_y1, max_samples,
     cpu_nodes_usage_aggregate[i][0] = counters_y1[i][1];
     cpu_nodes_usage_aggregate[i][1] = 0;
   }
+  var read_write_metrics_aggregate = new Array(counters_y2.length);
+  for (var i = 0; i < counters_y2.length; ++i) {
+    read_write_metrics_aggregate[i] = new Array(max_samples.allocated + 2)
+        .fill(null);
+    read_write_metrics_aggregate[i][0] = counters_y2[i][1];
+    read_write_metrics_aggregate[i][1] = 0;
+  }
   var sampled_utilization_timeseries = new Array(max_samples.allocated + 2)
       .fill(null);
   sampled_utilization_timeseries[0] = timeaxis_name;
   mapTimeseriesCounters(parent_profile.child_profiles[0].time_series_counters, counters_y1);
-  return {cpu_nodes_usage_aggregate, sampled_utilization_timeseries};
+  mapTimeseriesCounters(parent_profile.child_profiles[0].time_series_counters, counters_y2);
+  return {cpu_nodes_usage_aggregate, read_write_metrics_aggregate,
+      sampled_utilization_timeseries};
 }
 
-export function getUtilizationHeight() {
+function getUtilizationHeight() {
   return Math.max(diagram_min_height, window.innerHeight *
       host_utilization_resize_factor);
 }
 
+export function getUtilizationWrapperHeight() {
+  return (utilization_metrics_parse_successful && host_utilization_visible) *
+      (getUtilizationHeight() + diagram_controls_height);
+}
+
+export function resetUtilizationHeight() {
+  host_utilization_resize_factor = 0.2;
+}
+
 export function toogleUtilizationVisibility() {
   if (utilization_metrics_parse_successful && host_utilization_visible) {
-    host_utilization_diagram.style.display = "initial";
+    host_utilization_wrapper.style.display = "initial";
   } else {
-    host_utilization_diagram.style.display = "none";
+    host_utilization_wrapper.style.display = "none";
   }
 }
 
 export async function resizeUtilizationChart() {
   if (host_utilization_chart == null) return;
   var chart_width = diagram_width - margin_chart_end - name_width;
+  host_utilization_resize_bar.style.marginLeft = `${name_width + chart_width / 4}px`;
+  host_utilization_resize_bar.style.width = `${chart_width / 2}px`;
+  host_utilization_resize_bar.style.marginRight = `${chart_width / 4}px`;
   host_utilization_chart.resize({
     height : getUtilizationHeight(),
     width : diagram_width
@@ -170,8 +229,9 @@ export function collectUtilizationFromProfile() {
     var per_node_profiles = profile.child_profiles[2].child_profiles[0];
     console.assert(per_node_profiles.profile_name == "Per Node Profiles");
     if (host_utilization_chart == null) {
-      ({cpu_nodes_usage_aggregate, sampled_utilization_timeseries} =
+      ({cpu_nodes_usage_aggregate, read_write_metrics_aggregate,
-          initializeUtilizationMetrics(per_node_profiles, cpu_utilization_counters,
+          sampled_utilization_timeseries} = initializeUtilizationMetrics(
+          per_node_profiles, cpu_utilization_counters, read_write_metrics_counters,
           max_samples_utilization, utilization_timeaxis_name));
       initializeUtilizationChart();
     }
@@ -187,9 +247,11 @@ export function collectUtilizationFromProfile() {
     // For each node's profile seperately aggregate into two different units of metrics
     aggregateProfileTimeseries(per_node_profiles, cpu_nodes_usage_aggregate,
         cpu_utilization_counters, max_samples_utilization);
+    aggregateProfileTimeseries(per_node_profiles, read_write_metrics_aggregate,
+        read_write_metrics_counters, max_samples_utilization);
 
     // display warnings in case less samples are available without plotting
-    if (max_samples_utilization.available <= 2) {
+    if (max_samples_utilization.available < min_num_samples) {
       var utilization_samples_message = `Warning: Not enough samples for
           CPU utilization plot. Please decrease the value of starting flag
           variable <b><i>periodic_counter_update_period_ms </b></i> to
@@ -203,21 +265,32 @@ export function collectUtilizationFromProfile() {
     }
     // average the aggregated metrics
     cpu_nodes_usage_aggregate.forEach(function (acc_usage) {
-      for (var i = 2; i < max_samples_utilization.available + 2; ++i) {
+      for (var i = array_values_start_index; i < max_samples_utilization.available
+          + array_values_start_index; ++i) {
         acc_usage[i] = acc_usage[i] / (100 * per_node_profiles.child_profiles.length);
       }
     });
+    read_write_metrics_aggregate.forEach(function (acc_usage) {
+      for (var i = array_values_start_index; i < max_samples_utilization.available
+          + array_values_start_index; ++i) {
+        acc_usage[i] = acc_usage[i] / per_node_profiles.child_profiles.length;
+      }
+    });
     // generate timestamps for utilization values and decide on last timestamp value
     generateTimesamples(sampled_utilization_timeseries, max_samples_utilization, false);
 
     // load the utilization values to the chart
     host_utilization_chart.load({
-      columns : [...cpu_nodes_usage_aggregate, sampled_utilization_timeseries]
+      columns : [...cpu_nodes_usage_aggregate, ...read_write_metrics_aggregate,
+          sampled_utilization_timeseries]
     });
     // clear utilization value and timestamp samples arrays
     cpu_nodes_usage_aggregate.forEach(function (acc_usage) {
       clearTimeseriesValues(acc_usage, max_samples_utilization);
     });
+    read_write_metrics_aggregate.forEach(function (acc_usage) {
+      clearTimeseriesValues(acc_usage, max_samples_utilization);
+    });
     prev_utilization_num_samples = profile.child_profiles[1].summary_stats_counters[0]
         .num_of_samples;
     utilization_metrics_parse_successful = true;
@@ -229,6 +302,28 @@ export function collectUtilizationFromProfile() {
   }
 }
 
+host_utilization_resize_bar.addEventListener('mousedown',
+    function dragResizeBarBegin(mousedown_e) {
+  host_utilization_resize_bar.removeEventListener('mousedown', dragResizeBarBegin);
+  document.body.addEventListener('mousemove', dragResizeBar);
+  document.body.addEventListener('mouseup', function dragResrizeBarEnd() {
+    document.body.removeEventListener('mouseup', dragResrizeBarEnd);
+    document.body.removeEventListener('mousemove', dragResizeBar);
+    host_utilization_resize_bar.addEventListener('mousedown', dragResizeBarBegin);
+  });
+});
+
+
+host_utilization_close_btn.addEventListener('click', function(e) {
+  host_utilization_visible = false;
+  host_utilization_chart = destroyChart(host_utilization_chart, host_utilization_diagram);
+  toogleUtilizationVisibility();
+  setTimingDiagramDimensions();
+});
+
+host_utilization_close_btn.style.height = `${diagram_controls_height}px`;
+host_utilization_close_btn.style.fontSize = `${diagram_controls_height / 2}px`;
+
 if (typeof process != "undefined" && process.env.NODE_ENV === 'test') {
   exportedForTest = {initializeUtilizationMetrics};
 }

www/scripts/tests/query_timeline/fragment_metrics_diagram.test.js

@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import {exportedForTest} from "../../query_timeline/fragment_metrics_diagram.js";
+
+describe("Test initializeFragmentMetrics", () => {
+  // Test whether aggregate arrays and time sample arrays are correctly allocated
+  // based on counters and max_samples
+  var {initializeFragmentMetrics} = exportedForTest;
+  test("Basic Test", () => {
+    var parent_profile =
+    {
+      "profile_name": "Coordinator Fragment F31",
+      "num_children": 1,
+      "child_profiles": [
+        {
+          "profile_name": "Instance fe45c9c56d1:efd1b2a70000 (host=host-1:27000)",
+          "time_series_counters": [{
+            "counter_name": "MemoryUsage",
+            "unit": "BYTES",
+            "num": 6,
+            "period": 500,
+            "data": "12288,12288,12288,12288,12288,12288"
+          }, {
+            "counter_name": "ThreadUsage",
+            "unit": "UNIT",
+            "num": 1,
+            "period": 500,
+            "data": "4"
+          }]
+        }
+      ]
+    };
+    var max_samples = {
+      allocated : 4,
+      period : 0,
+      available : 0,
+      collected : 0
+    };
+    var counters = [
+        ["MemoryUsage", "memory usage", 0],
+        ["ThreadUsage", "thread usage", 0]
+    ];
+    var timeaxis_name = "fragment metrics timeticks";
+    var {fragment_metrics_aggregate, sampled_fragment_metrics_timeseries}
+        = initializeFragmentMetrics(parent_profile, counters, max_samples, timeaxis_name);
+    expect(fragment_metrics_aggregate).toEqual([
+        [null, 0, null, null, null, null, null],
+        [null, 0, null, null, null, null, null]
+    ]);
+    expect(sampled_fragment_metrics_timeseries).toEqual(
+        [timeaxis_name, null, null, null, null, null, null]
+    );
+  });
+});

www/scripts/tests/query_timeline/host_utilization_diagram.test.js

@@ -42,6 +42,12 @@ describe("Test initializeUtilizationMetrics", () => {
             "num": 59,
             "period": 100,
             "data": "312,679,445,440,301,301,312,125,125,437"
+          }, {
+            "counter_name": "HostNetworkRx",
+            "unit": "BASIS_POINTS",
+            "num": 59,
+            "period": 100,
+            "data": "312,679,445,440,301,301,312,125,125,437"
           }]
         }
       ]
@@ -56,18 +62,26 @@ describe("Test initializeUtilizationMetrics", () => {
         ["HostCpuUserPercentage", "avg io wait", 0],
         ["HostCpuSysPercentage", "avg sys", 0]
     ];
+    var counters_y2 = [
+        ["HostNetworkRx", "avg network rx", 0]
+    ];
     var timeaxis_name = "utilization timeticks";
-    var {cpu_nodes_usage_aggregate, sampled_utilization_timeseries} =
+    var {cpu_nodes_usage_aggregate, read_write_metrics_aggregate,
-        initializeUtilizationMetrics(parent_profile, counters_y1, max_samples,
+        sampled_utilization_timeseries} = initializeUtilizationMetrics(
-        timeaxis_name);
+        parent_profile, counters_y1, counters_y2,
+        max_samples, timeaxis_name);
     expect(cpu_nodes_usage_aggregate).toEqual([
         [counters_y1[0][1], 0, null, null, null],
         [counters_y1[1][1], 0, null, null, null]
     ]);
+    expect(read_write_metrics_aggregate).toEqual([
+        [counters_y2[0][1], 0, null, null, null]
+    ]);
     expect(sampled_utilization_timeseries).toEqual(
         [timeaxis_name, null, null, null, null]
     );
     expect(counters_y1[0][2]).toBe(0);
     expect(counters_y1[1][2]).toBe(1);
+    expect(counters_y2[0][2]).toBe(2);
   });
 });