You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@tez.apache.org by GitBox <gi...@apache.org> on 2022/05/30 14:35:58 UTC
[GitHub] [tez] johnnyCake1 opened a new pull request, #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
johnnyCake1 opened a new pull request, #212:
URL: https://github.com/apache/tez/pull/212
Add a /prof profiler endpoint like HiveServer2 has
- added 2 new endpoints to webuiservice:
- _**/prof**_ runs async-profiler
- _**/prof-output**_ serves the output file
- added corresponding tests in TestAM
- checked manually in browser
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r885393906
##########
tez-common/src/main/java/org/apache/tez/common/web/ProfileServlet.java:
##########
@@ -0,0 +1,326 @@
+package org.apache.tez.common.web;
+
+import com.google.common.base.Joiner;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class ProfileServlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
+ private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
+ private static final String ALLOWED_METHODS = "GET";
+ private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
+ private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
+ private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
+ private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
+ private static final String PROFILER_SCRIPT = "/profiler.sh";
+ private static final int DEFAULT_DURATION_SECONDS = 10;
+ private static final AtomicInteger ID_GEN = new AtomicInteger(0);
+ public static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
+
+ enum Event {
+ CPU("cpu"),
+ ALLOC("alloc"),
+ LOCK("lock"),
+ PAGE_FAULTS("page-faults"),
+ CONTEXT_SWITCHES("context-switches"),
+ CYCLES("cycles"),
+ INSTRUCTIONS("instructions"),
+ CACHE_REFERENCES("cache-references"),
+ CACHE_MISSES("cache-misses"),
+ BRANCHES("branches"),
+ BRANCH_MISSES("branch-misses"),
+ BUS_CYCLES("bus-cycles"),
+ L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
+ LLC_LOAD_MISSES("LLC-load-misses"),
+ DTLB_LOAD_MISSES("dTLB-load-misses"),
+ MEM_BREAKPOINT("mem:breakpoint"),
+ TRACE_TRACEPOINT("trace:tracepoint"),
+ ;
+
+ private final String internalName;
+
+ Event(final String internalName) {
+ this.internalName = internalName;
+ }
+
+ public String getInternalName() {
+ return internalName;
+ }
+
+ public static Event fromInternalName(final String name) {
+ for (Event event : values()) {
+ if (event.getInternalName().equalsIgnoreCase(name)) {
+ return event;
+ }
+ }
+ return null;
+ }
+ }
+
+ enum Output {
+ SUMMARY,
+ TRACES,
+ FLAT,
+ COLLAPSED,
+ SVG,
+ TREE,
+ JFR
+ }
+
+ private final Lock profilerLock = new ReentrantLock();
+ private Integer pid;
+ private String asyncProfilerHome;
+ private transient Process process;
+
+ public ProfileServlet() {
+ this.asyncProfilerHome = getAsyncProfilerHome();
+ this.pid = getPid();
+ LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome);
+ }
+ private Integer getPid(){
+ String pidStr = null;
+ String name = ManagementFactory.getRuntimeMXBean().getName();
+ if (name != null) {
+ int idx = name.indexOf("@");
+ if (idx != -1) {
+ pidStr = name.substring(0, name.indexOf("@"));
+ }
+ }
+ try {
+ if (pidStr != null) {
+ return Integer.valueOf(pidStr);
+ }
+ } catch (NumberFormatException nfe) {
+ // ignore
+ }
+ return null;
+ }
+
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ response.setContentType("text/plain; charset=UTF-8");
+ PrintStream out = new PrintStream(response.getOutputStream(), false, "UTF-8");
+ if (!HttpServer2.isInstrumentationAccessAllowed(this.getServletContext(), request, response)) {
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ setResponseHeader(response);
+ out.println("Unauthorized: Instrumentation access is not allowed!");
+ out.close();
+ return;
+ }
+
+ // make sure async profiler home is set
+ if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ setResponseHeader(response);
+ out.println("ASYNC_PROFILER_HOME env is not set");
+ out.close();
+ return;
+ }
+
+ // if pid is explicitly specified, use it else default to current process
+ pid = getInteger(request, "pid", pid);
+ // if pid is not specified in query param and if current process pid cannot be determined
+ if (pid == null) {
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ setResponseHeader(response);
+ out.println("'pid' query parameter unspecified or unable to determine PID of current process.");
+ out.close();
+ return;
+ }
+
+ final int duration = getInteger(request, "duration", DEFAULT_DURATION_SECONDS);
+ final Output output = getOutput(request);
+ final Event event = getEvent(request);
+ final Long interval = getLong(request, "interval");
+ final Integer jstackDepth = getInteger(request, "jstackdepth", null);
+ final Long bufsize = getLong(request, "bufsize");
+ final boolean thread = request.getParameterMap().containsKey("thread");
+ final boolean simple = request.getParameterMap().containsKey("simple");
+ final Integer width = getInteger(request, "width", null);
+ final Integer height = getInteger(request, "height", null);
+ final Double minwidth = getMinWidth(request);
+ final boolean reverse = request.getParameterMap().containsKey("reverse");
+ if (process == null || !process.isAlive()) {
+ try {
+ int lockTimeoutSecs = 3;
+ if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
+ try {
+ File outputFile = new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" +
+ event.name().toLowerCase() + "-" + ID_GEN.incrementAndGet() + "." +
+ output.name().toLowerCase());
+ List<String> cmd = new ArrayList<>();
+ cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
+ cmd.add("-e");
+ cmd.add(event.getInternalName());
+ cmd.add("-d");
+ cmd.add("" + duration);
+ cmd.add("-o");
+ cmd.add(output.name().toLowerCase());
+ cmd.add("-f");
+ cmd.add(outputFile.getAbsolutePath());
+ if (interval != null) {
+ cmd.add("-i");
+ cmd.add(interval.toString());
+ }
+ if (jstackDepth != null) {
+ cmd.add("-j");
+ cmd.add(jstackDepth.toString());
+ }
+ if (bufsize != null) {
+ cmd.add("-b");
+ cmd.add(bufsize.toString());
+ }
+ if (thread) {
+ cmd.add("-t");
+ }
+ if (simple) {
+ cmd.add("-s");
+ }
+ if (width != null) {
+ cmd.add("--width");
+ cmd.add(width.toString());
+ }
+ if (height != null) {
+ cmd.add("--height");
+ cmd.add(height.toString());
+ }
+ if (minwidth != null) {
+ cmd.add("--minwidth");
+ cmd.add(minwidth.toString());
+ }
+ if (reverse) {
+ cmd.add("--reverse");
+ }
+ cmd.add(pid.toString());
+ process = new ProcessBuilder(cmd).start();
+
+ // set response and set refresh header to output location
+ setResponseHeader(response);
+ response.setStatus(HttpServletResponse.SC_ACCEPTED);
+ String relativeUrl = "/prof-output";
+ // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified via url param
+ int refreshDelay = getInteger(request, "refreshDelay", 0);
+ // instead of sending redirect, set auto-refresh so that browsers will refresh with redirected url
+ response.setHeader("Refresh", (duration + refreshDelay) + "; URL=" + relativeUrl + '?' + ProfileOutputServlet.fileQueryParam + '=' + outputFile.getName());
+
+ out.println("Profiled PID: " + pid);
+ out.println("Given duration : " + duration);
+ out.println("Output_dir: " + OUTPUT_DIR);
+ out.println();
+ out.println("Started [" + event.getInternalName() + "] profiling. This page will automatically redirect to " +
+ relativeUrl + " after " + duration + " seconds.\n\ncommand:\n" + Joiner.on(" ").join(cmd));
+ out.println("Refreshing to: " + relativeUrl);
+ out.println("Status :" + response.getStatus());
+ out.flush();
+ } finally {
+ profilerLock.unlock();
+ }
+ } else {
+ setResponseHeader(response);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ out.println("Unable to acquire lock. Another instance of profiler might be running.");
+ LOG.warn("Unable to acquire lock in {} seconds. Another instance of profiler might be running.",
+ lockTimeoutSecs);
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while acquiring profile lock.", e);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ }
+ } else {
+ setResponseHeader(response);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ out.println("Another instance of profiler is already running.");
+ }
+ out.close();
+ }
+
+ private Integer getInteger(final HttpServletRequest req, final String param, final Integer defaultValue) {
+ final String value = req.getParameter(param);
+ if (value != null) {
+ try {
+ return Integer.valueOf(value);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+ return defaultValue;
+ }
+
+ private Long getLong(final HttpServletRequest req, final String param) {
+ final String value = req.getParameter(param);
+ if (value != null) {
+ try {
+ return Long.valueOf(value);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private Double getMinWidth(final HttpServletRequest req) {
+ final String value = req.getParameter("minwidth");
+ if (value != null) {
+ try {
+ return Double.valueOf(value);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private Event getEvent(final HttpServletRequest req) {
+ final String eventArg = req.getParameter("event");
+ if (eventArg != null) {
+ Event event = Event.fromInternalName(eventArg);
+ return event == null ? Event.CPU : event;
+ }
+ return Event.CPU;
+ }
+
+ private Output getOutput(final HttpServletRequest req) {
+ final String outputArg = req.getParameter("output");
+ if (req.getParameter("output") != null) {
Review Comment:
use outputArg variable here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1144716117
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 13m 0s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 5m 59s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 28s | master passed |
| +1 :green_heart: | compile | 2m 35s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 2m 27s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 29s | master passed |
| +1 :green_heart: | javadoc | 2m 31s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 2m 19s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 1m 4s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 59s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 12s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 4s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | compile | 1m 1s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 1s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 45s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 57s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 52s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 2m 27s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 33s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 31s | tez-dag in the patch passed. |
| -1 :x: | unit | 36m 1s | tez-tests in the patch failed. |
| +1 :green_heart: | asflicense | 0m 50s | The patch does not generate ASF License warnings. |
| | | 97m 45s | |
| Reason | Tests |
|-------:|:------|
| Failed junit tests | tez.test.TestRecovery |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/7/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 19e80a4ffb59 4.15.0-58-generic #64-Ubuntu SMP Tue Aug 6 11:12:41 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / cf9e3ff30 |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| unit | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/7/artifact/out/patch-unit-tez-tests.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/7/testReport/ |
| Max. process+thread count | 1342 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/7/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1150255086
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 20m 55s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 6m 4s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 13m 21s | master passed |
| +1 :green_heart: | compile | 3m 24s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 3m 14s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 3m 4s | master passed |
| +1 :green_heart: | javadoc | 3m 12s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 2m 53s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 1m 8s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 5m 24s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 38s | the patch passed |
| +1 :green_heart: | compile | 1m 52s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 52s | the patch passed |
| +1 :green_heart: | compile | 1m 39s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 39s | the patch passed |
| +1 :green_heart: | checkstyle | 1m 13s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 1m 23s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 17s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 3m 48s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 42s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 59s | tez-dag in the patch passed. |
| -1 :x: | unit | 19m 24s | tez-tests in the patch failed. |
| +0 :ok: | asflicense | 1m 8s | ASF License check generated no output? |
| | | 103m 29s | |
| Reason | Tests |
|-------:|:------|
| Failed junit tests | tez.test.TestAMRecovery |
| | tez.test.TestDAGRecovery2 |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/8/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 48e0ffa6e135 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / e5a557802 |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| unit | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/8/artifact/out/patch-unit-tez-tests.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/8/testReport/ |
| Max. process+thread count | 808 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/8/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1150751958
recovery failures are most probably unrelated
left very minor comments, other than that, looks good to me, this is very close to be committed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r885383247
##########
tez-common/src/main/java/org/apache/tez/common/web/ProfileOutputServlet.java:
##########
@@ -0,0 +1,52 @@
+package org.apache.tez.common.web;
+
+import org.apache.hadoop.yarn.webapp.MimeType;
+import org.eclipse.jetty.servlet.DefaultServlet;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+public class ProfileOutputServlet extends DefaultServlet {
+ public static final String fileQueryParam = "file";
+ public ProfileOutputServlet() {
+
+ }
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ String queriedFile = request.getParameter(fileQueryParam);
+ if (queriedFile == null) {
+ response.setContentType(MimeType.TEXT);
Review Comment:
you can refactor these lines to a single method (message string as parameter) and reuse it below
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r885386692
##########
tez-common/src/main/java/org/apache/tez/common/web/ProfileServlet.java:
##########
@@ -0,0 +1,326 @@
+package org.apache.tez.common.web;
+
+import com.google.common.base.Joiner;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class ProfileServlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
+ private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
+ private static final String ALLOWED_METHODS = "GET";
+ private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
+ private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
+ private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
+ private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
+ private static final String PROFILER_SCRIPT = "/profiler.sh";
+ private static final int DEFAULT_DURATION_SECONDS = 10;
+ private static final AtomicInteger ID_GEN = new AtomicInteger(0);
+ public static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
+
+ enum Event {
+ CPU("cpu"),
+ ALLOC("alloc"),
+ LOCK("lock"),
+ PAGE_FAULTS("page-faults"),
+ CONTEXT_SWITCHES("context-switches"),
+ CYCLES("cycles"),
+ INSTRUCTIONS("instructions"),
+ CACHE_REFERENCES("cache-references"),
+ CACHE_MISSES("cache-misses"),
+ BRANCHES("branches"),
+ BRANCH_MISSES("branch-misses"),
+ BUS_CYCLES("bus-cycles"),
+ L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
+ LLC_LOAD_MISSES("LLC-load-misses"),
+ DTLB_LOAD_MISSES("dTLB-load-misses"),
+ MEM_BREAKPOINT("mem:breakpoint"),
+ TRACE_TRACEPOINT("trace:tracepoint"),
+ ;
+
+ private final String internalName;
+
+ Event(final String internalName) {
+ this.internalName = internalName;
+ }
+
+ public String getInternalName() {
+ return internalName;
+ }
+
+ public static Event fromInternalName(final String name) {
+ for (Event event : values()) {
+ if (event.getInternalName().equalsIgnoreCase(name)) {
+ return event;
+ }
+ }
+ return null;
+ }
+ }
+
+ enum Output {
+ SUMMARY,
+ TRACES,
+ FLAT,
+ COLLAPSED,
+ SVG,
+ TREE,
+ JFR
+ }
+
+ private final Lock profilerLock = new ReentrantLock();
+ private Integer pid;
+ private String asyncProfilerHome;
+ private transient Process process;
+
+ public ProfileServlet() {
+ this.asyncProfilerHome = getAsyncProfilerHome();
+ this.pid = getPid();
+ LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome);
+ }
+ private Integer getPid(){
Review Comment:
please refactor this to a public static utility method, e.g. to TezUtilsInternal
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1144715389
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 16m 55s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 5m 52s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 42s | master passed |
| +1 :green_heart: | compile | 2m 7s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 1m 57s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 59s | master passed |
| +1 :green_heart: | javadoc | 2m 2s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 46s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 0m 56s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 26s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 2s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | compile | 0m 59s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 59s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 43s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 55s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 48s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 2m 31s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 32s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 21s | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 37m 36s | tez-tests in the patch passed. |
| +1 :green_heart: | asflicense | 0m 41s | The patch does not generate ASF License warnings. |
| | | 99m 43s | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/6/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 119da864a5b5 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / cf9e3ff30 |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/6/testReport/ |
| Max. process+thread count | 1293 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/6/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r885391487
##########
tez-tests/src/test/java/org/apache/tez/test/TestAM.java:
##########
@@ -76,6 +76,12 @@ public static void setup() throws IOException {
tezClusterConf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000);
tezClusterConf.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 1000);
tezClusterConf.set(YarnConfiguration.PROXY_ADDRESS, "localhost");
+ //provide temporary profiler script to test /prof end point
+ String profilerHome = System.getProperty("java.io.tmpdir");
Review Comment:
refactor this to a method (profilerHome + profiler file), like, getProfiler
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1151302087
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 1m 30s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 6m 25s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 11m 12s | master passed |
| +1 :green_heart: | compile | 2m 12s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 2m 2s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 58s | master passed |
| +1 :green_heart: | javadoc | 2m 15s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 52s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 1m 0s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 45s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 11s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 10s | the patch passed |
| +1 :green_heart: | compile | 1m 20s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 20s | the patch passed |
| +1 :green_heart: | compile | 1m 8s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 8s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 49s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 1m 2s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 1s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 3m 12s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 35s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 26s | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 42m 0s | tez-tests in the patch passed. |
| +1 :green_heart: | asflicense | 0m 52s | The patch does not generate ASF License warnings. |
| | | 93m 29s | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/9/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 613648d0f4ce 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / e5a557802 |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/9/testReport/ |
| Max. process+thread count | 1221 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/9/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r893142614
##########
tez-common/src/main/java/org/apache/tez/common/web/ProfileOutputServlet.java:
##########
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.common.web;
+
+import org.apache.hadoop.yarn.webapp.MimeType;
+import org.eclipse.jetty.servlet.DefaultServlet;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * Servlet to serve files generated by {@link ProfileServlet}.
+ */
+public class ProfileOutputServlet extends DefaultServlet {
+ public static final String FILE_QUERY_PARAM = "file";
+
+ public ProfileOutputServlet() {
Review Comment:
is this needed to be explicitly defined here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog merged pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog merged PR #212:
URL: https://github.com/apache/tez/pull/212
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1151316725
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 16m 36s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 5m 57s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 40s | master passed |
| +1 :green_heart: | compile | 2m 8s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 1m 58s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 0s | master passed |
| +1 :green_heart: | javadoc | 2m 3s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 49s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 0m 56s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 27s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 3s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | compile | 0m 59s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 59s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 43s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 55s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 48s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 2m 34s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 32s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 2s | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 37m 0s | tez-tests in the patch passed. |
| +1 :green_heart: | asflicense | 0m 41s | The patch does not generate ASF License warnings. |
| | | 99m 38s | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/10/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux eeeef1bf9c31 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / e5a557802 |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/10/testReport/ |
| Max. process+thread count | 1195 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/10/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r893143898
##########
tez-tests/src/test/java/org/apache/tez/test/TestAM.java:
##########
@@ -74,8 +74,13 @@ public static void setup() throws IOException {
Configuration tezClusterConf = new Configuration();
tezClusterConf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
tezClusterConf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000);
- tezClusterConf.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 1000);
+ tezClusterConf.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 2000);
tezClusterConf.set(YarnConfiguration.PROXY_ADDRESS, "localhost");
+ //provide temporary profiler script to test /prof end point
Review Comment:
nit: "endpoint"
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1151310717
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 1m 48s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 6m 8s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 11m 35s | master passed |
| +1 :green_heart: | compile | 2m 19s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 2m 8s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 9s | master passed |
| +1 :green_heart: | javadoc | 2m 12s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 59s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 1m 2s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 54s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 17s | the patch passed |
| +1 :green_heart: | compile | 1m 27s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 27s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 46s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 1m 1s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 53s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 2m 49s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 34s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 10s | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 40m 54s | tez-tests in the patch passed. |
| +1 :green_heart: | asflicense | 0m 42s | The patch does not generate ASF License warnings. |
| | | 92m 35s | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/11/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux e71ecda8f3bf 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / e5a557802 |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/11/testReport/ |
| Max. process+thread count | 1234 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/11/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r885392096
##########
tez-tests/src/test/java/org/apache/tez/test/TestAM.java:
##########
@@ -133,14 +144,18 @@ public void testAMWebUIService() throws TezException, IOException, InterruptedEx
}
private void checkAddress(String url) {
+ checkAddress(url, 200);
+ }
+
+ private void checkAddress(String url, int expectedCode) {
boolean success = false;
try {
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
connection.connect();
- success = (connection.getResponseCode() == 200);
+ success = (connection.getResponseCode() == expectedCode);
} catch (Exception e) {
LOG.error("Error while checking url: " + url, e);
}
assertTrue(url + " should be available", success);
}
-}
+}
Review Comment:
nit, new line is expected at the end of classes I guess
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1143994732
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 1m 18s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 5m 58s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 56s | master passed |
| +1 :green_heart: | compile | 2m 13s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 2m 1s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 5s | master passed |
| +1 :green_heart: | javadoc | 2m 12s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 51s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 0m 56s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 37s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 3s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | compile | 0m 59s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 59s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 43s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| -1 :x: | javadoc | 0m 15s | tez-common-jdkPrivateBuild-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
| +1 :green_heart: | javadoc | 0m 50s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | findbugs | 2m 34s | the patch passed |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 32s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 24s | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 38m 37s | tez-tests in the patch passed. |
| +1 :green_heart: | asflicense | 0m 42s | The patch does not generate ASF License warnings. |
| | | 86m 17s | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/5/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux ca2365550bad 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / 24a77c93f |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| javadoc | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/5/artifact/out/diff-javadoc-javadoc-tez-common-jdkPrivateBuild-11.0.15+10-Ubuntu-0ubuntu0.20.04.1.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/5/testReport/ |
| Max. process+thread count | 1187 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/5/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1142221710
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 18m 36s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 6m 9s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 51s | master passed |
| +1 :green_heart: | compile | 2m 15s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 2m 2s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 6s | master passed |
| +1 :green_heart: | javadoc | 2m 12s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 53s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 0m 57s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 38s | master passed |
| -0 :warning: | patch | 1m 23s | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 2s | the patch passed |
| +1 :green_heart: | compile | 1m 10s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 10s | the patch passed |
| +1 :green_heart: | compile | 1m 0s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 0s | the patch passed |
| -0 :warning: | checkstyle | 0m 16s | tez-common: The patch generated 268 new + 56 unchanged - 0 fixed = 324 total (was 56) |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 56s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 50s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| -1 :x: | findbugs | 0m 40s | tez-common generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 33s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 21s | tez-dag in the patch passed. |
| -1 :x: | unit | 39m 29s | tez-tests in the patch failed. |
| -1 :x: | asflicense | 0m 54s | The patch generated 2 ASF License warnings. |
| | | 105m 6s | |
| Reason | Tests |
|-------:|:------|
| FindBugs | module:tez-common |
| | HTTP parameter directly written to HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:[line 36] |
| | Relative path traversal in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HttpServletResponse) At ProfileOutputServlet.java:[line 29] |
| Failed junit tests | tez.test.TestAM |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 7f8646ac0162 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / 24a77c93f |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| checkstyle | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/artifact/out/diff-checkstyle-tez-common.txt |
| findbugs | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/artifact/out/new-findbugs-tez-common.html |
| unit | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/artifact/out/patch-unit-tez-tests.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/testReport/ |
| asflicense | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/artifact/out/patch-asflicense-problems.txt |
| Max. process+thread count | 1171 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/2/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1142281809
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 17m 8s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 5m 53s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 55s | master passed |
| +1 :green_heart: | compile | 2m 8s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 1m 58s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 59s | master passed |
| +1 :green_heart: | javadoc | 2m 5s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 47s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 0m 55s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 25s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 11s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 2s | the patch passed |
| +1 :green_heart: | compile | 1m 8s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 8s | the patch passed |
| +1 :green_heart: | compile | 1m 0s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 0s | the patch passed |
| -0 :warning: | checkstyle | 0m 17s | tez-common: The patch generated 268 new + 56 unchanged - 0 fixed = 324 total (was 56) |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 55s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 50s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| -1 :x: | findbugs | 0m 39s | tez-common generated 2 new + 0 unchanged - 0 fixed = 2 total (was 0) |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 45s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 21s | tez-dag in the patch passed. |
| -1 :x: | unit | 35m 31s | tez-tests in the patch failed. |
| -1 :x: | asflicense | 0m 41s | The patch generated 2 ASF License warnings. |
| | | 98m 31s | |
| Reason | Tests |
|-------:|:------|
| FindBugs | module:tez-common |
| | HTTP parameter directly written to HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:[line 36] |
| | Relative path traversal in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HttpServletResponse) At ProfileOutputServlet.java:[line 29] |
| Failed junit tests | tez.test.TestAM |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 52f798bd90b0 4.15.0-175-generic #184-Ubuntu SMP Thu Mar 24 17:48:36 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / 24a77c93f |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| checkstyle | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/artifact/out/diff-checkstyle-tez-common.txt |
| findbugs | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/artifact/out/new-findbugs-tez-common.html |
| unit | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/artifact/out/patch-unit-tez-tests.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/testReport/ |
| asflicense | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/artifact/out/patch-asflicense-problems.txt |
| Max. process+thread count | 1137 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/3/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r893140138
##########
tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java:
##########
@@ -322,6 +323,25 @@ public static <T extends Enum<T>> Set<T> getEnums(Configuration conf, String con
return enums;
}
+ public static Integer getPid() {
+ String pidStr = null;
+ String name = ManagementFactory.getRuntimeMXBean().getName();
+ if (name != null) {
+ int idx = name.indexOf("@");
+ if (idx != -1) {
+ pidStr = name.substring(0, name.indexOf("@"));
+ }
+ }
+ try {
+ if (pidStr != null) {
+ return Integer.valueOf(pidStr);
+ }
+ } catch (NumberFormatException nfe) {
+ // ignore
Review Comment:
instead of ignoring it, can we put an INFO level message containing pidStr?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r893140424
##########
tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java:
##########
@@ -7,7 +7,7 @@
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
Review Comment:
this change was not intentional I guess :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1141316590
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 12m 56s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 5m 56s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 26s | master passed |
| +1 :green_heart: | compile | 1m 48s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 1m 36s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 29s | master passed |
| +1 :green_heart: | javadoc | 1m 46s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 1m 23s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 0m 45s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 2m 58s | master passed |
| -0 :warning: | patch | 1m 4s | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 9s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 1s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | compile | 0m 57s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 0m 57s | the patch passed |
| -0 :warning: | checkstyle | 0m 16s | tez-common: The patch generated 285 new + 0 unchanged - 0 fixed = 285 total (was 0) |
| -0 :warning: | checkstyle | 0m 12s | tez-tests: The patch generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 50s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 48s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| -1 :x: | findbugs | 0m 38s | tez-common generated 3 new + 0 unchanged - 0 fixed = 3 total (was 0) |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 32s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 32s | tez-dag in the patch passed. |
| -1 :x: | unit | 38m 44s | tez-tests in the patch failed. |
| -1 :x: | asflicense | 0m 35s | The patch generated 2 ASF License warnings. |
| | | 94m 20s | |
| Reason | Tests |
|-------:|:------|
| FindBugs | module:tez-common |
| | HTTP parameter directly written to HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:[line 43] |
| | Relative path traversal in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HttpServletResponse) At ProfileOutputServlet.java:[line 30] |
| | HTTP parameter written to Servlet output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:[line 34] |
| Failed junit tests | tez.test.TestAM |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux fc4678634afd 4.15.0-169-generic #177-Ubuntu SMP Thu Feb 3 10:50:38 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / 24a77c93f |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| checkstyle | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/artifact/out/diff-checkstyle-tez-common.txt |
| checkstyle | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/artifact/out/diff-checkstyle-tez-tests.txt |
| findbugs | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/artifact/out/new-findbugs-tez-common.html |
| unit | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/artifact/out/patch-unit-tez-tests.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/testReport/ |
| asflicense | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/artifact/out/patch-asflicense-problems.txt |
| Max. process+thread count | 1385 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/1/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] abstractdog commented on a diff in pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
abstractdog commented on code in PR #212:
URL: https://github.com/apache/tez/pull/212#discussion_r885393307
##########
tez-common/src/main/java/org/apache/tez/common/web/ProfileServlet.java:
##########
@@ -0,0 +1,326 @@
+package org.apache.tez.common.web;
+
+import com.google.common.base.Joiner;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class ProfileServlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
+ private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
+ private static final String ALLOWED_METHODS = "GET";
+ private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
+ private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
+ private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
+ private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
+ private static final String PROFILER_SCRIPT = "/profiler.sh";
+ private static final int DEFAULT_DURATION_SECONDS = 10;
+ private static final AtomicInteger ID_GEN = new AtomicInteger(0);
+ public static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output";
+
+ enum Event {
+ CPU("cpu"),
+ ALLOC("alloc"),
+ LOCK("lock"),
+ PAGE_FAULTS("page-faults"),
+ CONTEXT_SWITCHES("context-switches"),
+ CYCLES("cycles"),
+ INSTRUCTIONS("instructions"),
+ CACHE_REFERENCES("cache-references"),
+ CACHE_MISSES("cache-misses"),
+ BRANCHES("branches"),
+ BRANCH_MISSES("branch-misses"),
+ BUS_CYCLES("bus-cycles"),
+ L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
+ LLC_LOAD_MISSES("LLC-load-misses"),
+ DTLB_LOAD_MISSES("dTLB-load-misses"),
+ MEM_BREAKPOINT("mem:breakpoint"),
+ TRACE_TRACEPOINT("trace:tracepoint"),
+ ;
+
+ private final String internalName;
+
+ Event(final String internalName) {
+ this.internalName = internalName;
+ }
+
+ public String getInternalName() {
+ return internalName;
+ }
+
+ public static Event fromInternalName(final String name) {
+ for (Event event : values()) {
+ if (event.getInternalName().equalsIgnoreCase(name)) {
+ return event;
+ }
+ }
+ return null;
+ }
+ }
+
+ enum Output {
+ SUMMARY,
+ TRACES,
+ FLAT,
+ COLLAPSED,
+ SVG,
+ TREE,
+ JFR
+ }
+
+ private final Lock profilerLock = new ReentrantLock();
+ private Integer pid;
+ private String asyncProfilerHome;
+ private transient Process process;
+
+ public ProfileServlet() {
+ this.asyncProfilerHome = getAsyncProfilerHome();
+ this.pid = getPid();
+ LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome);
+ }
+ private Integer getPid(){
+ String pidStr = null;
+ String name = ManagementFactory.getRuntimeMXBean().getName();
+ if (name != null) {
+ int idx = name.indexOf("@");
+ if (idx != -1) {
+ pidStr = name.substring(0, name.indexOf("@"));
+ }
+ }
+ try {
+ if (pidStr != null) {
+ return Integer.valueOf(pidStr);
+ }
+ } catch (NumberFormatException nfe) {
+ // ignore
+ }
+ return null;
+ }
+
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ response.setContentType("text/plain; charset=UTF-8");
+ PrintStream out = new PrintStream(response.getOutputStream(), false, "UTF-8");
+ if (!HttpServer2.isInstrumentationAccessAllowed(this.getServletContext(), request, response)) {
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ setResponseHeader(response);
+ out.println("Unauthorized: Instrumentation access is not allowed!");
+ out.close();
+ return;
+ }
+
+ // make sure async profiler home is set
+ if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ setResponseHeader(response);
+ out.println("ASYNC_PROFILER_HOME env is not set");
+ out.close();
+ return;
+ }
+
+ // if pid is explicitly specified, use it else default to current process
+ pid = getInteger(request, "pid", pid);
+ // if pid is not specified in query param and if current process pid cannot be determined
+ if (pid == null) {
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ setResponseHeader(response);
+ out.println("'pid' query parameter unspecified or unable to determine PID of current process.");
+ out.close();
+ return;
+ }
+
+ final int duration = getInteger(request, "duration", DEFAULT_DURATION_SECONDS);
+ final Output output = getOutput(request);
+ final Event event = getEvent(request);
+ final Long interval = getLong(request, "interval");
+ final Integer jstackDepth = getInteger(request, "jstackdepth", null);
+ final Long bufsize = getLong(request, "bufsize");
+ final boolean thread = request.getParameterMap().containsKey("thread");
+ final boolean simple = request.getParameterMap().containsKey("simple");
+ final Integer width = getInteger(request, "width", null);
+ final Integer height = getInteger(request, "height", null);
+ final Double minwidth = getMinWidth(request);
+ final boolean reverse = request.getParameterMap().containsKey("reverse");
+ if (process == null || !process.isAlive()) {
+ try {
+ int lockTimeoutSecs = 3;
+ if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
+ try {
+ File outputFile = new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" +
+ event.name().toLowerCase() + "-" + ID_GEN.incrementAndGet() + "." +
+ output.name().toLowerCase());
+ List<String> cmd = new ArrayList<>();
+ cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
+ cmd.add("-e");
+ cmd.add(event.getInternalName());
+ cmd.add("-d");
+ cmd.add("" + duration);
+ cmd.add("-o");
+ cmd.add(output.name().toLowerCase());
+ cmd.add("-f");
+ cmd.add(outputFile.getAbsolutePath());
+ if (interval != null) {
+ cmd.add("-i");
+ cmd.add(interval.toString());
+ }
+ if (jstackDepth != null) {
+ cmd.add("-j");
+ cmd.add(jstackDepth.toString());
+ }
+ if (bufsize != null) {
+ cmd.add("-b");
+ cmd.add(bufsize.toString());
+ }
+ if (thread) {
+ cmd.add("-t");
+ }
+ if (simple) {
+ cmd.add("-s");
+ }
+ if (width != null) {
+ cmd.add("--width");
+ cmd.add(width.toString());
+ }
+ if (height != null) {
+ cmd.add("--height");
+ cmd.add(height.toString());
+ }
+ if (minwidth != null) {
+ cmd.add("--minwidth");
+ cmd.add(minwidth.toString());
+ }
+ if (reverse) {
+ cmd.add("--reverse");
+ }
+ cmd.add(pid.toString());
+ process = new ProcessBuilder(cmd).start();
+
+ // set response and set refresh header to output location
+ setResponseHeader(response);
+ response.setStatus(HttpServletResponse.SC_ACCEPTED);
+ String relativeUrl = "/prof-output";
+ // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified via url param
+ int refreshDelay = getInteger(request, "refreshDelay", 0);
+ // instead of sending redirect, set auto-refresh so that browsers will refresh with redirected url
+ response.setHeader("Refresh", (duration + refreshDelay) + "; URL=" + relativeUrl + '?' + ProfileOutputServlet.fileQueryParam + '=' + outputFile.getName());
+
+ out.println("Profiled PID: " + pid);
+ out.println("Given duration : " + duration);
+ out.println("Output_dir: " + OUTPUT_DIR);
+ out.println();
+ out.println("Started [" + event.getInternalName() + "] profiling. This page will automatically redirect to " +
+ relativeUrl + " after " + duration + " seconds.\n\ncommand:\n" + Joiner.on(" ").join(cmd));
+ out.println("Refreshing to: " + relativeUrl);
+ out.println("Status :" + response.getStatus());
+ out.flush();
+ } finally {
+ profilerLock.unlock();
+ }
+ } else {
+ setResponseHeader(response);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ out.println("Unable to acquire lock. Another instance of profiler might be running.");
+ LOG.warn("Unable to acquire lock in {} seconds. Another instance of profiler might be running.",
+ lockTimeoutSecs);
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while acquiring profile lock.", e);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ }
+ } else {
+ setResponseHeader(response);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ out.println("Another instance of profiler is already running.");
+ }
+ out.close();
+ }
+
+ private Integer getInteger(final HttpServletRequest req, final String param, final Integer defaultValue) {
+ final String value = req.getParameter(param);
+ if (value != null) {
+ try {
+ return Integer.valueOf(value);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+ return defaultValue;
+ }
+
+ private Long getLong(final HttpServletRequest req, final String param) {
+ final String value = req.getParameter(param);
+ if (value != null) {
+ try {
+ return Long.valueOf(value);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private Double getMinWidth(final HttpServletRequest req) {
+ final String value = req.getParameter("minwidth");
+ if (value != null) {
+ try {
+ return Double.valueOf(value);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private Event getEvent(final HttpServletRequest req) {
+ final String eventArg = req.getParameter("event");
+ if (eventArg != null) {
+ Event event = Event.fromInternalName(eventArg);
+ return event == null ? Event.CPU : event;
+ }
+ return Event.CPU;
+ }
+
+ private Output getOutput(final HttpServletRequest req) {
+ final String outputArg = req.getParameter("output");
+ if (req.getParameter("output") != null) {
+ try {
+ return Output.valueOf(outputArg.trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return Output.SVG;
Review Comment:
put a WARN log message here, indicating that output value is invalid, returning with default SVG
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tez] tez-yetus commented on pull request #212: TEZ-4038: Add a /prof profiler endpoint like HiveServer2 has
Posted by GitBox <gi...@apache.org>.
tez-yetus commented on PR #212:
URL: https://github.com/apache/tez/pull/212#issuecomment-1143432154
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Comment |
|:----:|----------:|--------:|:--------|
| +0 :ok: | reexec | 13m 16s | Docker mode activated. |
||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. |
| +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. |
||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 6m 9s | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 18s | master passed |
| +1 :green_heart: | compile | 2m 33s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | compile | 2m 27s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 2m 31s | master passed |
| +1 :green_heart: | javadoc | 2m 32s | master passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 2m 17s | master passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +0 :ok: | spotbugs | 1m 5s | Used deprecated FindBugs config; considering switching to SpotBugs. |
| +1 :green_heart: | findbugs | 3m 48s | master passed |
||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 11s | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 3s | the patch passed |
| +1 :green_heart: | compile | 1m 9s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javac | 1m 9s | the patch passed |
| +1 :green_heart: | compile | 1m 2s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 1m 2s | the patch passed |
| +1 :green_heart: | checkstyle | 0m 47s | the patch passed |
| +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. |
| +1 :green_heart: | javadoc | 0m 57s | the patch passed with JDK Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 |
| +1 :green_heart: | javadoc | 0m 50s | the patch passed with JDK Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| -1 :x: | findbugs | 0m 41s | tez-common generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 33s | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 31s | tez-dag in the patch passed. |
| -1 :x: | unit | 42m 46s | tez-tests in the patch failed. |
| -1 :x: | asflicense | 0m 52s | The patch generated 2 ASF License warnings. |
| | | 104m 46s | |
| Reason | Tests |
|-------:|:------|
| FindBugs | module:tez-common |
| | HTTP parameter directly written to HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:HTTP header output in org.apache.tez.common.web.ProfileOutputServlet.doGet(HttpServletRequest, HttpServletResponse) At ProfileOutputServlet.java:[line 35] |
| Failed junit tests | tez.test.TestAM |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/4/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/212 |
| JIRA Issue | TEZ-4038 |
| Optional Tests | dupname asflicense javac javadoc unit spotbugs findbugs checkstyle compile |
| uname | Linux 4a971196784a 4.15.0-112-generic #113-Ubuntu SMP Thu Jul 9 23:41:39 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | personality/tez.sh |
| git revision | master / 24a77c93f |
| Default Java | Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Private Build-11.0.15+10-Ubuntu-0ubuntu0.20.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_312-8u312-b07-0ubuntu1~20.04-b07 |
| findbugs | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/4/artifact/out/new-findbugs-tez-common.html |
| unit | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/4/artifact/out/patch-unit-tez-tests.txt |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/4/testReport/ |
| asflicense | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/4/artifact/out/patch-asflicense-problems.txt |
| Max. process+thread count | 1288 (vs. ulimit of 5500) |
| modules | C: tez-common tez-dag tez-tests U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-212/4/console |
| versions | git=2.25.1 maven=3.6.3 findbugs=3.0.1 |
| Powered by | Apache Yetus 0.12.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org