You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vi...@apache.org on 2018/02/19 09:25:31 UTC
[3/7] drill git commit: DRILL-6140: Correctly list Operators in
Profiles Page
DRILL-6140: Correctly list Operators in Profiles Page
Operators listed in Profiles Page don't always correspond with operator specified in Physical Plan.
This commit fixes that by using the PhysicalPlan as a reference, but reverts to the inferred names in the event of an Exchange-based operator
closes #1116
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/58f3b104
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/58f3b104
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/58f3b104
Branch: refs/heads/master
Commit: 58f3b10464f5a3b969ce19705750cb163be9b287
Parents: 3314905
Author: Kunal Khatua <kk...@maprtech.com>
Authored: Tue Feb 6 22:14:58 2018 -0800
Committer: Vitalii Diravka <vi...@gmail.com>
Committed: Fri Feb 16 20:21:06 2018 +0000
----------------------------------------------------------------------
.../server/rest/profile/OperatorWrapper.java | 17 ++++++++++++++--
.../server/rest/profile/ProfileWrapper.java | 21 +++++++++++++++++++-
2 files changed, 35 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/58f3b104/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 6322435..0df062f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -45,12 +45,25 @@ public class OperatorWrapper {
private final String operatorName;
private final int size;
- public OperatorWrapper(int major, List<ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String>> opsAndHostsList) {
+ public OperatorWrapper(int major, List<ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String>> opsAndHostsList, Map<String, String> phyOperMap) {
Preconditions.checkArgument(opsAndHostsList.size() > 0);
this.major = major;
firstProfile = opsAndHostsList.get(0).getLeft().getLeft();
operatorType = CoreOperatorType.valueOf(firstProfile.getOperatorType());
- operatorName = operatorType == null ? UNKNOWN_OPERATOR : operatorType.toString();
+ //Update Name from Physical Map
+ String path = new OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
+ //Use Plan Extracted Operator Names if available
+ String extractedOpName = phyOperMap.get(path);
+ String inferredOpName = operatorType == null ? UNKNOWN_OPERATOR : operatorType.toString();
+ //Revert to inferred names for exceptional cases
+ // 1. Extracted 'FLATTEN' operator is NULL
+ // 2. Extracted 'SCAN' could be a PARQUET_ROW_GROUP_SCAN, or KAFKA_SUB_SCAN, or etc.
+ // 3. Extracted 'UNION_EXCHANGE' could be a SINGLE_SENDER or UNORDERED_RECEIVER
+ if (extractedOpName == null || inferredOpName.contains(extractedOpName) || extractedOpName.endsWith("_EXCHANGE")) {
+ operatorName = inferredOpName;
+ } else {
+ operatorName = extractedOpName;
+ }
this.opsAndHosts = opsAndHostsList;
size = opsAndHostsList.size();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/58f3b104/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
index d59b464..2c964da 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.CaseFormat;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.drill.common.config.DrillConfig;
@@ -57,10 +58,13 @@ public class ProfileWrapper {
private final long majorFragmentTallyTotal;
private final OptionList options;
private final boolean onlyImpersonationEnabled;
+ private Map<String, String> physicalOperatorMap;
public ProfileWrapper(final QueryProfile profile, DrillConfig drillConfig) {
this.profile = profile;
this.id = QueryIdHelper.getQueryId(profile.getId());
+ //Generating Operator Name map (DRILL-6140)
+ generateOpMap(profile.getPlan());
final List<FragmentWrapper> fragmentProfiles = new ArrayList<>();
@@ -107,7 +111,7 @@ public class ProfileWrapper {
Collections.sort(keys);
for (final ImmutablePair<Integer, Integer> ip : keys) {
- ows.add(new OperatorWrapper(ip.getLeft(), opmap.get(ip)));
+ ows.add(new OperatorWrapper(ip.getLeft(), opmap.get(ip), physicalOperatorMap));
}
this.operatorProfiles = ows;
@@ -322,4 +326,19 @@ public class ProfileWrapper {
public boolean isOnlyImpersonationEnabled() {
return onlyImpersonationEnabled;
}
+
+ //Generates operator names inferred from physical plan
+ private void generateOpMap(String plan) {
+ this.physicalOperatorMap = new HashMap<String,String>();
+ //[e.g ] operatorLine = "01-03 Flatten(flattenField=[$1]) : rowType = RecordType(ANY rfsSpecCode, ..."
+ String[] operatorLine = plan.split("\\n");
+ for (String line : operatorLine) {
+ String[] lineToken = line.split("\\s+", 3);
+ //[e.g ] operatorPath = "01-xx-03"
+ String operatorPath = lineToken[0].trim().replaceFirst("-", "-xx-"); //Required format for lookup
+ //[e.g ] extractedOperatorName = "FLATTEN"
+ String extractedOperatorName = CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, lineToken[1].split("\\(", 2)[0].trim());
+ physicalOperatorMap.put(operatorPath, extractedOperatorName);
+ }
+ }
}