You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/16 23:31:22 UTC
[13/32] git commit: DRILL-980: Add profile summary to web ui
DRILL-980: Add profile summary to web ui
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/49a9ff27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/49a9ff27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/49a9ff27
Branch: refs/heads/master
Commit: 49a9ff27f283cbc1c8749989ff408440a0275e7d
Parents: 6801a5d
Author: Steven Phillips <sp...@maprtech.com>
Authored: Fri Jun 13 00:17:47 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:02:36 2014 -0700
----------------------------------------------------------------------
.../exec/server/rest/ProfileResources.java | 4 +-
.../drill/exec/server/rest/ProfileWrapper.java | 148 +++++++++++++++++++
.../drill/exec/work/foreman/FragmentData.java | 2 +-
.../work/fragment/AbstractStatusReporter.java | 1 +
.../src/main/resources/rest/profile/profile.ftl | 10 +-
5 files changed, 160 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
index 386db46..7d29ea3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
@@ -57,7 +57,9 @@ public class ProfileResources {
QueryProfile profile = profiles.get(queryId);
if(profile == null) profile = QueryProfile.getDefaultInstance();
- return new Viewable("/rest/profile/profile.ftl", profile);
+ ProfileWrapper wrapper = new ProfileWrapper(profile);
+
+ return new Viewable("/rest/profile/profile.ftl", wrapper);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
new file mode 100644
index 0000000..a1d4df9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile;
+import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile;
+import org.apache.drill.exec.proto.UserBitShared.OperatorProfile;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+
+import java.text.NumberFormat;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+public class ProfileWrapper {
+
+ NumberFormat format = NumberFormat.getInstance(Locale.US);
+
+ public QueryProfile profile;
+
+ public ProfileWrapper(QueryProfile profile) {
+ this.profile = profile;
+ }
+
+ public QueryProfile getProfile() {
+ return profile;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("MAJOR FRAGMENTS\nid\tmin\tavg\tmax\t(time in ms)\n\n" + listMajorFragments());
+ builder.append("\n");
+ for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) {
+ builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), new MajorFragmentWrapper(majorProfile).toString()));
+ }
+ return builder.toString();
+ }
+
+ public String listMajorFragments() {
+ StringBuilder builder = new StringBuilder();
+ for (MajorFragmentProfile m : profile.getFragmentProfileList()) {
+ List<Long> totalTimes = Lists.newArrayList();
+ for (MinorFragmentProfile minorFragmentProfile : m.getMinorFragmentProfileList()) {
+ totalTimes.add(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime());
+ }
+ long min = Collections.min(totalTimes);
+ long max = Collections.max(totalTimes);
+ long sum = 0;
+ for (Long l : totalTimes) {
+ sum += l;
+ }
+ long avg = sum / totalTimes.size();
+ builder.append(String.format("%d\t%s\t%s\t%s\n", m.getMajorFragmentId(), format.format(min), format.format(avg), format.format(max)));
+ }
+ return builder.toString();
+ }
+
+ public class MajorFragmentWrapper {
+ MajorFragmentProfile majorFragmentProfile;
+
+ public MajorFragmentWrapper(MajorFragmentProfile majorFragmentProfile) {
+ this.majorFragmentProfile = majorFragmentProfile;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Minor Fragments\nid\ttotal time (ms)\n%s\nOperators\nid\ttype\tmin\tavg\tmax\t(time in ns)\n%s\n", new MinorFragmentsInMajor().toString(), new OperatorsInMajor().toString());
+ }
+
+ public class MinorFragmentsInMajor {
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ for (MinorFragmentProfile minorFragmentProfile: majorFragmentProfile.getMinorFragmentProfileList()) {
+ builder.append(String.format("%d\t%s\n", minorFragmentProfile.getMinorFragmentId(), format.format(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime())));
+ }
+ return builder.toString();
+ }
+ }
+
+ public class OperatorsInMajor {
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ int numOperators = majorFragmentProfile.getMinorFragmentProfile(0).getOperatorProfileCount();
+ int numFragments = majorFragmentProfile.getMinorFragmentProfileCount();
+ long[][] values = new long[numOperators + 1][numFragments];
+ CoreOperatorType[] operatorTypes = new CoreOperatorType[numOperators + 1];
+
+ for (int i = 0; i < numFragments; i++) {
+ MinorFragmentProfile minorProfile = majorFragmentProfile.getMinorFragmentProfile(i);
+ for (int j = 0; j < numOperators; j++) {
+ OperatorProfile operatorProfile = minorProfile.getOperatorProfile(j);
+ int operatorId = operatorProfile.getOperatorId();
+ values[operatorId][i] = operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos();
+ if (i == 0) {
+ operatorTypes[operatorId] = CoreOperatorType.valueOf(operatorProfile.getOperatorType());
+ }
+ }
+ }
+
+ for (int j = 0; j < numOperators + 1; j++) {
+ if (operatorTypes[j] == null) {
+ continue;
+ }
+ long min = Long.MAX_VALUE;
+ long max = Long.MIN_VALUE;
+ long sum = 0;
+
+ for (int i = 0; i < numFragments; i++) {
+ min = Math.min(min, values[j][i]);
+ max = Math.max(max, values[j][i]);
+ sum += values[j][i];
+ }
+
+ long avg = sum / numFragments;
+
+ builder.append(String.format("%d\t%s\t%s\t%s\t%s\n", j, operatorTypes[j].toString(), format.format(min), format.format(avg), format.format(max)));
+ }
+ return builder.toString();
+ }
+ }
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
index d9dd33e..b9b3de0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
@@ -31,7 +31,7 @@ public class FragmentData {
public FragmentData(FragmentHandle handle, DrillbitEndpoint endpoint, boolean isLocal) {
super();
- MinorFragmentProfile f = MinorFragmentProfile.newBuilder().setState(FragmentState.SENDING).build();
+ MinorFragmentProfile f = MinorFragmentProfile.newBuilder().setState(FragmentState.SENDING).setMinorFragmentId(handle.getMinorFragmentId()).build();
this.status = FragmentStatus.newBuilder().setHandle(handle).setProfile(f).build();
this.endpoint = endpoint;
this.isLocal = isLocal;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
index 30e4b6a..8d9da19 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
@@ -48,6 +48,7 @@ public abstract class AbstractStatusReporter implements StatusReporter{
}
status.setHandle(context.getHandle());
b.setMemoryUsed(context.getAllocator().getAllocatedMemory());
+ b.setMinorFragmentId(context.getHandle().getMinorFragmentId());
status.setProfile(b);
return status;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/resources/rest/profile/profile.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
index ffd22db..c3149ab 100644
--- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
@@ -20,7 +20,7 @@
<h2>Query</h2>
<form role="form" action="/query" method="POST">
<div class="form-group">
- <textarea class="form-control" id="query" name="query">${model.query}</textarea>
+ <textarea class="form-control" id="query" name="query">${model.getProfile().query}</textarea>
</div>
<div class="form-group">
<div class="radio-inline">
@@ -47,11 +47,15 @@
<div class="page-header">
<h2>Physical Plan</h2>
</div>
- <p><pre>${model.plan}</pre></p>
+ <p><pre>${model.profile.plan}</pre></p>
<div class="page-header">
- <h2>Complete Profile</h2>
+ <h2>Profile Summary</h2>
</div>
<p><pre>${model.toString()}</pre></p>
+ <div class="page-header">
+ <h2>Complete Profile</h2>
+ </div>
+ <p><pre>${model.profile.toString()}</pre></p>
</#macro>
<@page_html/>