You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ay...@apache.org on 2024/09/23 11:57:32 UTC
(hive) branch branch-4.0 updated (63e204d36e2 -> 00abddba415)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a change to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
from 63e204d36e2 HIVE-28173: Fixed staging dir issues on materialized views on HDFS encrypted tables (Stephen Carlin reviewed by Krisztian Kasa, Denys Kuzmenko)
new 823f8d26ff9 HIVE-28309: fixed Task Execution Summary duration is not displayed (#5285) (Ryu Kobayashi reviewed by Butao Zhang, Laszlo Bodor)
new 00abddba415 HIVE-28305: ProfileServlet: add html to output formats and prepare for profiler 3.0 (#5281) (Laszlo Bodor reviewed by Denys Kuzmenko, Ayush Saxena, Kokila N)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hive/http/ProfileServlet.java | 35 +++++++++++++++-------
.../ql/exec/tez/monitoring/RenderStrategy.java | 12 ++++----
.../hive/ql/exec/tez/monitoring/TezJobMonitor.java | 6 ++--
3 files changed, 34 insertions(+), 19 deletions(-)
(hive) 01/02: HIVE-28309: fixed Task Execution Summary duration is not displayed (#5285) (Ryu Kobayashi reviewed by Butao Zhang, Laszlo Bodor)
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 823f8d26ff9fd5caa5317ca2964b6cc9ae2de408
Author: Ryu Kobayashi <be...@gmail.com>
AuthorDate: Tue Jun 11 16:54:08 2024 +0900
HIVE-28309: fixed Task Execution Summary duration is not displayed (#5285) (Ryu Kobayashi reviewed by Butao Zhang, Laszlo Bodor)
---
.../hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java | 12 ++++++------
.../hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java | 6 +++---
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java
index c58e0c73c8c..132489e4906 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java
@@ -51,14 +51,14 @@ class RenderStrategy {
private long lastPrintTime = 0L;
private String lastReport = null;
- BaseUpdateFunction(TezJobMonitor monitor) {
+ BaseUpdateFunction(TezJobMonitor monitor, PerfLogger perfLogger) {
this.monitor = monitor;
print_interval = HiveConf.getTimeVar(
monitor.getHiveConf(),
HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL,
TimeUnit.MILLISECONDS
);
- perfLogger = SessionState.getPerfLogger();
+ this.perfLogger = perfLogger;
}
@Override
@@ -155,8 +155,8 @@ class RenderStrategy {
SessionState.get().getConf().getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_INPLACE_PROGRESS);
private final ZoneId localTimeZone = SessionState.get().getConf().getLocalTimeZone();
- LogToFileFunction(TezJobMonitor monitor) {
- super(monitor);
+ LogToFileFunction(TezJobMonitor monitor, PerfLogger perfLogger) {
+ super(monitor, perfLogger);
}
@Override
@@ -186,8 +186,8 @@ class RenderStrategy {
*/
private final InPlaceUpdate inPlaceUpdate;
- InPlaceUpdateFunction(TezJobMonitor monitor) {
- super(monitor);
+ InPlaceUpdateFunction(TezJobMonitor monitor, PerfLogger perfLogger) {
+ super(monitor, perfLogger);
inPlaceUpdate = new InPlaceUpdate(SessionState.LogHelper.getInfoStream());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
index fbbbf189e48..aab13378c8e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
@@ -124,17 +124,17 @@ public class TezJobMonitor {
this.dag = dag;
this.context = ctx;
console = SessionState.getConsole();
+ this.perfLogger = perfLogger;
updateFunction = updateFunction();
this.counters = counters;
- this.perfLogger = perfLogger;
}
private RenderStrategy.UpdateFunction updateFunction() {
return InPlaceUpdate.canRenderInPlace(hiveConf)
&& !SessionState.getConsole().getIsSilent()
&& !SessionState.get().isHiveServerQuery()
- ? new RenderStrategy.InPlaceUpdateFunction(this)
- : new RenderStrategy.LogToFileFunction(this);
+ ? new RenderStrategy.InPlaceUpdateFunction(this, perfLogger)
+ : new RenderStrategy.LogToFileFunction(this, perfLogger);
}
private boolean isProfilingEnabled() {
(hive) 02/02: HIVE-28305: ProfileServlet: add html to output formats and prepare for profiler 3.0 (#5281) (Laszlo Bodor reviewed by Denys Kuzmenko, Ayush Saxena, Kokila N)
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 00abddba415cd545203c4a4ed2d6bf65f0404cec
Author: Bodor Laszlo <bo...@gmail.com>
AuthorDate: Thu Jun 6 14:41:30 2024 +0200
HIVE-28305: ProfileServlet: add html to output formats and prepare for profiler 3.0 (#5281) (Laszlo Bodor reviewed by Denys Kuzmenko, Ayush Saxena, Kokila N)
---
.../java/org/apache/hive/http/ProfileServlet.java | 35 +++++++++++++++-------
1 file changed, 25 insertions(+), 10 deletions(-)
diff --git a/common/src/java/org/apache/hive/http/ProfileServlet.java b/common/src/java/org/apache/hive/http/ProfileServlet.java
index c2ff2712938..c1260e4a3fd 100644
--- a/common/src/java/org/apache/hive/http/ProfileServlet.java
+++ b/common/src/java/org/apache/hive/http/ProfileServlet.java
@@ -17,6 +17,9 @@ package org.apache.hive.http;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -45,19 +48,19 @@ import com.google.common.base.Joiner;
* // -m method fully qualified method name: 'ClassName.methodName'
* // -t profile different threads separately
* // -s simple class names instead of FQN
- * // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr
+ * // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html
* // --width px SVG width pixels (integer)
* // --height px SVG frame height pixels (integer)
* // --minwidth px skip frames smaller than px (double)
* // --reverse generate stack-reversed FlameGraph / Call tree
* Example:
- * - To collect 30 second CPU profile of current process (returns FlameGraph svg)
+ * - To collect 10 second CPU profile of current process (returns FlameGraph html)
* curl "http://localhost:10002/prof"
* - To collect 1 minute CPU profile of current process and output in tree format (html)
* curl "http://localhost:10002/prof?output=tree&duration=60"
- * - To collect 30 second heap allocation profile of current process (returns FlameGraph svg)
+ * - To collect 10 second heap allocation profile of current process (returns FlameGraph html)
* curl "http://localhost:10002/prof?event=alloc"
- * - To collect lock contention profile of current process (returns FlameGraph svg)
+ * - To collect lock contention profile of current process (returns FlameGraph html)
* curl "http://localhost:10002/prof?event=lock"
* Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events)
* // Perf events:
@@ -89,7 +92,6 @@ public class ProfileServlet extends HttpServlet {
private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
- private static final String PROFILER_SCRIPT = "/profiler.sh";
private static final int DEFAULT_DURATION_SECONDS = 10;
private static final AtomicInteger ID_GEN = new AtomicInteger(0);
static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
@@ -141,7 +143,8 @@ public class ProfileServlet extends HttpServlet {
COLLAPSED,
SVG,
TREE,
- JFR
+ JFR,
+ HTML
}
private Lock profilerLock = new ReentrantLock();
@@ -213,7 +216,7 @@ public class ProfileServlet extends HttpServlet {
(method == null ? event.name().toLowerCase() : method) + "-" + ID_GEN.incrementAndGet() + "." +
output.name().toLowerCase());
List<String> cmd = new ArrayList<>();
- cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
+ cmd.add(getProfilerScriptPath());
cmd.add("-e");
cmd.add(method == null ? event.getInternalName() : method);
cmd.add("-d");
@@ -293,6 +296,16 @@ public class ProfileServlet extends HttpServlet {
}
}
+ /**
+ * Get the path of the profiler script to be executed.
+ * Before async-profiler 3.0, the script was named profiler.sh, and after 3.0 it's bin/asprof
+ * @return
+ */
+ private String getProfilerScriptPath() {
+ Path defaultPath = Paths.get(asyncProfilerHome + "/bin/asprof");
+ return Files.exists(defaultPath)? defaultPath.toString() : asyncProfilerHome + "/profiler.sh";
+ }
+
private Integer getInteger(final HttpServletRequest req, final String param, final Integer defaultValue) {
final String value = req.getParameter(param);
if (value != null) {
@@ -340,14 +353,16 @@ public class ProfileServlet extends HttpServlet {
private Output getOutput(final HttpServletRequest req) {
final String outputArg = req.getParameter("output");
+ String outputFormat = outputArg.trim().toUpperCase();
if (req.getParameter("output") != null) {
try {
- return Output.valueOf(outputArg.trim().toUpperCase());
+ return Output.valueOf(outputFormat);
} catch (IllegalArgumentException e) {
- return Output.SVG;
+ LOG.warn("Output format value '{}' is invalid, returning with default HTML", outputFormat);
+ return Output.HTML;
}
}
- return Output.SVG;
+ return Output.HTML;
}
private void setResponseHeader(final HttpServletResponse response) {