You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/02/21 15:33:31 UTC
[1/4] drill git commit: DRILL-5263: Prevent left NLJoin with non
scalar subqueries
Repository: drill
Updated Branches:
refs/heads/master 300e9349a -> 38f816a45
DRILL-5263: Prevent left NLJoin with non scalar subqueries
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3ba24fb0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3ba24fb0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3ba24fb0
Branch: refs/heads/master
Commit: 3ba24fb05aa5365baf8e44a9dd437dc57f46d9b9
Parents: 300e934
Author: Serhii-Harnyk <se...@gmail.com>
Authored: Mon Feb 13 14:30:27 2017 +0000
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Tue Feb 21 00:29:56 2017 -0800
----------------------------------------------------------------------
.../exec/physical/impl/join/JoinUtils.java | 20 +++++++++++------
.../exec/planner/common/DrillJoinRelBase.java | 15 ++-----------
.../planner/physical/NestedLoopJoinPrule.java | 6 ++---
.../physical/impl/join/TestNestedLoopJoin.java | 23 +++++++++++++++++---
4 files changed, 38 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3ba24fb0/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
index caa18be..1f84965 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinUtils.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,25 +28,19 @@ import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.drill.exec.physical.impl.common.Comparator;
import org.apache.drill.exec.planner.logical.DrillAggregateRel;
-import org.apache.drill.exec.planner.logical.DrillFilterRel;
-import org.apache.drill.exec.planner.logical.DrillProjectRel;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.JoinCondition;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.resolver.TypeCastRules;
import java.util.LinkedList;
import java.util.List;
-import com.google.common.collect.Lists;
-
public class JoinUtils {
public enum JoinCategory {
@@ -257,4 +251,16 @@ public class JoinUtils {
return JoinCategory.EQUALITY;
}
+ /**
+ * Utility method to check if a any of input RelNodes is provably scalar.
+ *
+ * @param left the RelNode to be inspected.
+ * @param right the RelNode to be inspected.
+ * @return Return true if any of the given RelNodes is provably scalar.
+ * Otherwise, return false
+ */
+ public static boolean hasScalarSubqueryInput(RelNode left, RelNode right) {
+ return isScalarSubquery(left) || isScalarSubquery(right);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3ba24fb0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
index 6bd0e9c..7ec92a1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,9 +28,7 @@ import org.apache.drill.exec.planner.cost.DrillCostBase;
import org.apache.drill.exec.physical.impl.join.JoinUtils;
import org.apache.drill.exec.physical.impl.join.JoinUtils.JoinCategory;
import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
-import org.apache.drill.exec.planner.cost.DrillRelOptCost;
import org.apache.drill.exec.planner.physical.PrelUtil;
-import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.RelNode;
@@ -68,7 +66,7 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode {
if (category == JoinCategory.CARTESIAN || category == JoinCategory.INEQUALITY) {
if (PrelUtil.getPlannerSettings(planner).isNestedLoopJoinEnabled()) {
if (PrelUtil.getPlannerSettings(planner).isNlJoinForScalarOnly()) {
- if (hasScalarSubqueryInput()) {
+ if (JoinUtils.hasScalarSubqueryInput(left, right)) {
return computeLogicalJoinCost(planner);
} else {
/*
@@ -204,13 +202,4 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode {
return costFactory.makeCost(buildRowCount + probeRowCount, cpuCost, 0, 0, memCost);
}
- private boolean hasScalarSubqueryInput() {
- if (JoinUtils.isScalarSubquery(this.getLeft())
- || JoinUtils.isScalarSubquery(this.getRight())) {
- return true;
- }
-
- return false;
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3ba24fb0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
index 2cda15a..bfb47d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -49,7 +49,7 @@ public class NestedLoopJoinPrule extends JoinPruleBase {
PlannerSettings settings) {
JoinRelType type = join.getJoinType();
- if (! (type == JoinRelType.INNER || type == JoinRelType.LEFT)) {
+ if (!(type == JoinRelType.INNER || (type == JoinRelType.LEFT && JoinUtils.hasScalarSubqueryInput(left, right)))) {
return false;
}
@@ -63,7 +63,7 @@ public class NestedLoopJoinPrule extends JoinPruleBase {
}
if (settings.isNlJoinForScalarOnly()) {
- if (JoinUtils.isScalarSubquery(left) || JoinUtils.isScalarSubquery(right)) {
+ if (JoinUtils.hasScalarSubqueryInput(left, right)) {
return true;
} else {
return false;
http://git-wip-us.apache.org/repos/asf/drill/blob/3ba24fb0/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
index fa28001..6210022 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,12 +19,14 @@
package org.apache.drill.exec.physical.impl.join;
import org.apache.drill.PlanTestBase;
-import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.common.util.TestTools;
-import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
import org.junit.Ignore;
import org.junit.Test;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.StringStartsWith.startsWith;
+
public class TestNestedLoopJoin extends PlanTestBase {
private static String nlpattern = "NestedLoopJoin";
@@ -253,4 +255,19 @@ public class TestNestedLoopJoin extends PlanTestBase {
test(ENABLE_HJ);
test(ENABLE_MJ);
}
+
+ @Test(expected = UserRemoteException.class)
+ public void testExceptionLeftNlJoin() throws Exception {
+ try {
+ test(DISABLE_NLJ_SCALAR);
+ test("select r.r_regionkey, n.n_nationkey from cp.`tpch/nation.parquet` n " +
+ " left join cp.`tpch/region.parquet` r on n.n_regionkey < r.r_regionkey where n.n_nationkey < 3");
+ } catch (UserRemoteException e) {
+ assertThat("No expected current \"UNSUPPORTED_OPERATION ERROR\"",
+ e.getMessage(), startsWith("UNSUPPORTED_OPERATION ERROR"));
+ throw e;
+ } finally {
+ test("alter session reset `planner.enable_nljoin_for_scalar_only`");
+ }
+ }
}
[3/4] drill git commit: DRILL-5242: The UI breaks when trying to
render profiles having unknown metrics
Posted by jn...@apache.org.
DRILL-5242: The UI breaks when trying to render profiles having unknown metrics
Skip any metrics whose metric ID is unknown, This prevents any ArrayIndexOutOfBoundsException from being thrown and breaking the UI rendering.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0b8ef9e7
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0b8ef9e7
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0b8ef9e7
Branch: refs/heads/master
Commit: 0b8ef9e71f71f41cdc2c6a07d01361ee45199516
Parents: a05d700
Author: Kunal Khatua <kk...@users.noreply.github.com>
Authored: Fri Feb 3 17:29:47 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Tue Feb 21 00:48:22 2017 -0800
----------------------------------------------------------------------
.../exec/server/rest/profile/OperatorWrapper.java | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/0b8ef9e7/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 1926655..5d18911 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -19,6 +19,8 @@ package org.apache.drill.exec.server.rest.profile;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -161,11 +163,18 @@ public class OperatorWrapper {
null);
final Number[] values = new Number[metricNames.length];
+ //Track new/Unknown Metrics
+ final Set<Integer> unknownMetrics = new TreeSet<Integer>();
for (final MetricValue metric : op.getMetricList()) {
- if (metric.hasLongValue()) {
- values[metric.getMetricId()] = metric.getLongValue();
- } else if (metric.hasDoubleValue()) {
- values[metric.getMetricId()] = metric.getDoubleValue();
+ if (metric.getMetricId() < metricNames.length) {
+ if (metric.hasLongValue()) {
+ values[metric.getMetricId()] = metric.getLongValue();
+ } else if (metric.hasDoubleValue()) {
+ values[metric.getMetricId()] = metric.getDoubleValue();
+ }
+ } else {
+ //Tracking unknown metric IDs
+ unknownMetrics.add(metric.getMetricId());
}
}
for (final Number value : values) {
[2/4] drill git commit: DRILL-5230: Translation of millisecond
duration into hours is incorrect
Posted by jn...@apache.org.
DRILL-5230: Translation of millisecond duration into hours is incorrect
Fixed invalid representation of readable elapsed time using `TimeUnit` class in JDK.
e.g. 4545 sec is now correctly translated as `1h15m` instead of `17h15m`
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a05d7002
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a05d7002
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a05d7002
Branch: refs/heads/master
Commit: a05d7002fca19f11656740bd5d3782fe29f2b36d
Parents: 3ba24fb
Author: Kunal Khatua <kk...@maprtech.com>
Authored: Sun Jan 29 23:08:12 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Tue Feb 21 00:37:53 2017 -0800
----------------------------------------------------------------------
.../server/rest/profile/FragmentWrapper.java | 14 +--
.../server/rest/profile/OperatorWrapper.java | 14 +--
.../server/rest/profile/ProfileResources.java | 38 ++----
.../server/rest/profile/ProfileWrapper.java | 2 +-
.../rest/profile/SimpleDurationFormat.java | 77 +++++++++++++
.../exec/server/rest/profile/TableBuilder.java | 36 +-----
.../drill/exec/server/TestDurationFormat.java | 115 +++++++++++++++++++
7 files changed, 217 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
index 2a8b564..ec0e2c8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
@@ -57,8 +57,6 @@ public class FragmentWrapper {
public static final int NUM_NULLABLE_OVERVIEW_COLUMNS = FRAGMENT_OVERVIEW_COLUMNS.length - 2;
public void addSummary(TableBuilder tb) {
- final String fmt = " (%d)";
-
// Use only minor fragments that have complete profiles
// Complete iff the fragment profile has at least one operator profile, and start and end times.
final List<MinorFragmentProfile> complete = new ArrayList<>(
@@ -75,13 +73,13 @@ public class FragmentWrapper {
final MinorFragmentProfile firstStart = Collections.min(complete, Comparators.startTime);
final MinorFragmentProfile lastStart = Collections.max(complete, Comparators.startTime);
- tb.appendMillis(firstStart.getStartTime() - start, String.format(fmt, firstStart.getMinorFragmentId()));
- tb.appendMillis(lastStart.getStartTime() - start, String.format(fmt, lastStart.getMinorFragmentId()));
+ tb.appendMillis(firstStart.getStartTime() - start, null);
+ tb.appendMillis(lastStart.getStartTime() - start, null);
final MinorFragmentProfile firstEnd = Collections.min(complete, Comparators.endTime);
final MinorFragmentProfile lastEnd = Collections.max(complete, Comparators.endTime);
- tb.appendMillis(firstEnd.getEndTime() - start, String.format(fmt, firstEnd.getMinorFragmentId()));
- tb.appendMillis(lastEnd.getEndTime() - start, String.format(fmt, lastEnd.getMinorFragmentId()));
+ tb.appendMillis(firstEnd.getEndTime() - start, null);
+ tb.appendMillis(lastEnd.getEndTime() - start, null);
long total = 0;
for (final MinorFragmentProfile p : complete) {
@@ -90,9 +88,9 @@ public class FragmentWrapper {
final MinorFragmentProfile shortRun = Collections.min(complete, Comparators.runTime);
final MinorFragmentProfile longRun = Collections.max(complete, Comparators.runTime);
- tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), String.format(fmt, shortRun.getMinorFragmentId()));
+ tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null);
tb.appendMillis(total / complete.size(), null);
- tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), String.format(fmt, longRun.getMinorFragmentId()));
+ tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null);
final MinorFragmentProfile lastUpdate = Collections.max(complete, Comparators.lastUpdate);
tb.appendTime(lastUpdate.getLastUpdate(), null);
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 4cc7971..1926655 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -32,8 +32,6 @@ import org.apache.drill.exec.proto.UserBitShared.StreamProfile;
* Wrapper class for profiles of ALL operator instances of the same operator type within a major fragment.
*/
public class OperatorWrapper {
- private static final String format = " (%s)";
-
private final int major;
private final List<ImmutablePair<OperatorProfile, Integer>> ops; // operator profile --> minor fragment number
private final OperatorProfile firstProfile;
@@ -114,21 +112,21 @@ public class OperatorWrapper {
final ImmutablePair<OperatorProfile, Integer> shortSetup = Collections.min(ops, Comparators.setupTime);
final ImmutablePair<OperatorProfile, Integer> longSetup = Collections.max(ops, Comparators.setupTime);
- tb.appendNanos(shortSetup.getLeft().getSetupNanos(), String.format(format, shortSetup.getRight()));
+ tb.appendNanos(shortSetup.getLeft().getSetupNanos(), null);
tb.appendNanos(Math.round(setupSum / size), null);
- tb.appendNanos(longSetup.getLeft().getSetupNanos(), String.format(format, longSetup.getRight()));
+ tb.appendNanos(longSetup.getLeft().getSetupNanos(), null);
final ImmutablePair<OperatorProfile, Integer> shortProcess = Collections.min(ops, Comparators.processTime);
final ImmutablePair<OperatorProfile, Integer> longProcess = Collections.max(ops, Comparators.processTime);
- tb.appendNanos(shortProcess.getLeft().getProcessNanos(), String.format(format, shortProcess.getRight()));
+ tb.appendNanos(shortProcess.getLeft().getProcessNanos(), null);
tb.appendNanos(Math.round(processSum / size), null);
- tb.appendNanos(longProcess.getLeft().getProcessNanos(), String.format(format, longProcess.getRight()));
+ tb.appendNanos(longProcess.getLeft().getProcessNanos(), null);
final ImmutablePair<OperatorProfile, Integer> shortWait = Collections.min(ops, Comparators.waitTime);
final ImmutablePair<OperatorProfile, Integer> longWait = Collections.max(ops, Comparators.waitTime);
- tb.appendNanos(shortWait.getLeft().getWaitNanos(), String.format(format, shortWait.getRight()));
+ tb.appendNanos(shortWait.getLeft().getWaitNanos(), null);
tb.appendNanos(Math.round(waitSum / size), null);
- tb.appendNanos(longWait.getLeft().getWaitNanos(), String.format(format, longWait.getRight()));
+ tb.appendNanos(longWait.getLeft().getWaitNanos(), null);
final ImmutablePair<OperatorProfile, Integer> peakMem = Collections.max(ops, Comparators.operatorPeakMemory);
tb.appendBytes(Math.round(memSum / size), null);
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
index 24d37b2..9ab4f4a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
@@ -68,25 +68,6 @@ public class ProfileResources {
@Inject DrillUserPrincipal principal;
@Inject SecurityContext sc;
- /**
- * Returns elapsed time a human-readable format. If end time is less than the start time, current epoch time is assumed as the end time.
- * e.g. getPrettyDuration(1468368841695,1468394096016) = '7 hr 00 min 54.321 sec'
- * @param startTimeMillis Start Time in milliseconds
- * @param endTimeMillis End Time in milliseconds
- * @return Human-Readable Elapsed Time
- */
- public static String getPrettyDuration(long startTimeMillis, long endTimeMillis) {
- long durationInMillis = (startTimeMillis > endTimeMillis ? System.currentTimeMillis() : endTimeMillis) - startTimeMillis;
- long hours = TimeUnit.MILLISECONDS.toHours(durationInMillis);
- long minutes = TimeUnit.MILLISECONDS.toMinutes(durationInMillis) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(durationInMillis));
- long seconds = TimeUnit.MILLISECONDS.toSeconds(durationInMillis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(durationInMillis));
- long milliSeconds = durationInMillis - TimeUnit.SECONDS.toMillis(TimeUnit.MILLISECONDS.toSeconds(durationInMillis));
- String formattedDuration = (hours > 0 ? hours + " hr " : "") +
- ((minutes + hours) > 0 ? String.format("%02d min ", minutes) : "") +
- seconds + "." + String.format("%03d sec", milliSeconds) ;
- return formattedDuration;
- }
-
public static class ProfileInfo implements Comparable<ProfileInfo> {
public static final SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
@@ -137,7 +118,7 @@ public class ProfileResources {
}
public String getDuration() {
- return getPrettyDuration(startTime, endTime);
+ return (new SimpleDurationFormat(startTime, endTime)).verbose();
}
public String getState() {
@@ -237,8 +218,8 @@ public class ProfileResources {
return new QProfiles(runningQueries, finishedQueries, errors);
} catch (Exception e) {
throw UserException.resourceError(e)
- .message("Failed to get profiles from persistent or ephemeral store.")
- .build(logger);
+ .message("Failed to get profiles from persistent or ephemeral store.")
+ .build(logger);
}
}
@@ -291,8 +272,8 @@ public class ProfileResources {
}
throw UserException.validationError()
- .message("No profile with given query id '%s' exists. Please verify the query id.", queryId)
- .build(logger);
+ .message("No profile with given query id '%s' exists. Please verify the query id.", queryId)
+ .build(logger);
}
@@ -352,16 +333,17 @@ public class ProfileResources {
private void checkOrThrowProfileViewAuthorization(final QueryProfile profile) {
if (!principal.canManageProfileOf(profile.getUser())) {
throw UserException.permissionError()
- .message("Not authorized to view the profile of query '%s'", profile.getId())
- .build(logger);
+ .message("Not authorized to view the profile of query '%s'", profile.getId())
+ .build(logger);
}
}
private void checkOrThrowQueryCancelAuthorization(final String queryUser, final String queryId) {
if (!principal.canManageQueryOf(queryUser)) {
throw UserException.permissionError()
- .message("Not authorized to cancel the query '%s'", queryId)
- .build(logger);
+ .message("Not authorized to cancel the query '%s'", queryId)
+ .build(logger);
}
}
}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
index 57223f6..20de752 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
@@ -115,7 +115,7 @@ public class ProfileWrapper {
}
public String getProfileDuration() {
- return ProfileResources.getPrettyDuration(profile.getStart(), profile.getEnd());
+ return (new SimpleDurationFormat(profile.getStart(), profile.getEnd())).verbose();
}
public String getQueryId() {
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/SimpleDurationFormat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/SimpleDurationFormat.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/SimpleDurationFormat.java
new file mode 100644
index 0000000..00f2b66
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/SimpleDurationFormat.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest.profile;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Representation of a millisecond duration in a human-readable format
+ */
+public class SimpleDurationFormat {
+ private long days;
+ private long hours;
+ private long minutes;
+ private long seconds;
+ private long milliSeconds;
+ private long durationInMillis;
+
+ //Block creation of any default objects
+ @SuppressWarnings("unused")
+ private SimpleDurationFormat() {}
+
+ /**
+ * If end time is less than the start time, current epoch time is assumed as the end time.
+ * @param startTimeMillis
+ * @param endTimeMillis
+ */
+ public SimpleDurationFormat(long startTimeMillis, long endTimeMillis) {
+ durationInMillis = (startTimeMillis > endTimeMillis ? System.currentTimeMillis() : endTimeMillis) - startTimeMillis;
+ days = TimeUnit.MILLISECONDS.toDays(durationInMillis);
+ hours = TimeUnit.MILLISECONDS.toHours(durationInMillis) - TimeUnit.DAYS.toHours(TimeUnit.MILLISECONDS.toDays(durationInMillis));
+ minutes = TimeUnit.MILLISECONDS.toMinutes(durationInMillis) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(durationInMillis));
+ seconds = TimeUnit.MILLISECONDS.toSeconds(durationInMillis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(durationInMillis));
+ milliSeconds = durationInMillis - TimeUnit.SECONDS.toMillis(TimeUnit.MILLISECONDS.toSeconds(durationInMillis));
+ }
+
+ /**
+ * Return a compact representation of elapsed time with only the most significant time units and no spaces
+ * @return duration
+ */
+ public String compact() {
+ if (days >= 1) {
+ return days + "d" + hours + "h" + minutes + "m";
+ } else if (hours >= 1) {
+ return hours + "h" + minutes + "m";
+ } else if (minutes >= 1) {
+ return minutes + "m" + seconds + "s";
+ } else {
+ return String.format("%.3fs", seconds + milliSeconds/1000.0);
+ }
+ }
+
+ /**
+ * Return a verbose representation of elapsed time down to millisecond granularity
+ * @return duration
+ */
+ public String verbose() {
+ return (days > 0 ? days + " day " : "") +
+ ((hours + days) > 0 ? hours + " hr " : "") +
+ ((minutes + hours + days) > 0 ? String.format("%02d min ", minutes) : "") +
+ seconds + "." + String.format("%03d sec", milliSeconds) ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java
index 8355797..7893be4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java
@@ -21,20 +21,10 @@ import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
-import java.util.Date;
import java.util.Locale;
-class TableBuilder {
+public class TableBuilder {
private final NumberFormat format = NumberFormat.getInstance(Locale.US);
- private final SimpleDateFormat days = new SimpleDateFormat("DD'd'hh'h'mm'm'");
- private final SimpleDateFormat sdays = new SimpleDateFormat("DD'd'hh'h'mm'm'");
- private final SimpleDateFormat hours = new SimpleDateFormat("HH'h'mm'm'");
- private final SimpleDateFormat shours = new SimpleDateFormat("H'h'mm'm'");
- private final SimpleDateFormat mins = new SimpleDateFormat("mm'm'ss's'");
- private final SimpleDateFormat smins = new SimpleDateFormat("m'm'ss's'");
-
- private final SimpleDateFormat secs = new SimpleDateFormat("ss.SSS's'");
- private final SimpleDateFormat ssecs = new SimpleDateFormat("s.SSS's'");
private final DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
private final DecimalFormat dec = new DecimalFormat("0.00");
private final DecimalFormat intformat = new DecimalFormat("#,###");
@@ -78,29 +68,7 @@ class TableBuilder {
}
public void appendMillis(final long p, final String link) {
- final double secs = p/1000.0;
- final double mins = secs/60;
- final double hours = mins/60;
- final double days = hours / 24;
- SimpleDateFormat timeFormat = null;
- if (days >= 10) {
- timeFormat = this.days;
- } else if (days >= 1) {
- timeFormat = this.sdays;
- } else if (hours >= 10) {
- timeFormat = this.hours;
- }else if(hours >= 1){
- timeFormat = this.shours;
- }else if (mins >= 10){
- timeFormat = this.mins;
- }else if (mins >= 1){
- timeFormat = this.smins;
- }else if (secs >= 10){
- timeFormat = this.secs;
- }else {
- timeFormat = this.ssecs;
- }
- appendCell(timeFormat.format(new Date(p)), null);
+ appendCell((new SimpleDurationFormat(0, p)).compact(), link);
}
public void appendNanos(final long p, final String link) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a05d7002/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDurationFormat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDurationFormat.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDurationFormat.java
new file mode 100644
index 0000000..f1d3037
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDurationFormat.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server;
+
+import org.apache.drill.exec.server.rest.profile.SimpleDurationFormat;
+import org.apache.drill.test.DrillTest;
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+/**
+ * Test translation of millisecond durations into human readable format
+ */
+public class TestDurationFormat extends DrillTest {
+ enum DurationFormat {
+ COMPACT,
+ VERBOSE
+ }
+
+ private void validateDurationFormat(long durationInMillisec, String expected, DurationFormat format) {
+ String formatted = null;
+ if (format.equals(DurationFormat.COMPACT)) {
+ formatted = new SimpleDurationFormat(0, durationInMillisec).compact();
+ }
+ else if (format.equals(DurationFormat.VERBOSE)) {
+ formatted = new SimpleDurationFormat(0, durationInMillisec).verbose();
+ }
+ assertEquals(formatted,expected);
+ }
+
+ @Test
+ public void testCompactTwoDigitMilliSec() {
+ validateDurationFormat(45, "0.045s", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseTwoDigitMilliSec() {
+ validateDurationFormat(45, "0.045 sec", DurationFormat.VERBOSE);
+ }
+
+ @Test
+ public void testCompactSecMillis() {
+ validateDurationFormat(4545, "4.545s", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseSecMillis() {
+ validateDurationFormat(4545, "4.545 sec", DurationFormat.VERBOSE);
+ }
+
+ @Test
+ public void testCompactMinSec() {
+ validateDurationFormat(454534, "7m34s", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseMinSec() {
+ validateDurationFormat(454534, "07 min 34.534 sec", DurationFormat.VERBOSE);
+ }
+
+ @Test
+ public void testCompactHourMin() {
+ validateDurationFormat(4545342, "1h15m", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseHourMin() {
+ validateDurationFormat(4545342, "1 hr 15 min 45.342 sec", DurationFormat.VERBOSE);
+ }
+
+ @Test
+ public void testCompactHalfDayHourMin() {
+ validateDurationFormat(45453420, "12h37m", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseHalfDayHourMin() {
+ validateDurationFormat(45453420, "12 hr 37 min 33.420 sec", DurationFormat.VERBOSE);
+ }
+
+ @Test
+ public void testCompactOneDayHourMin() {
+ validateDurationFormat(45453420 + 86400000, "1d12h37m", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseOneDayHourMin() {
+ validateDurationFormat(45453420 + 86400000, "1 day 12 hr 37 min 33.420 sec", DurationFormat.VERBOSE);
+ }
+
+ @Test
+ public void testCompactManyDayHourMin() {
+ validateDurationFormat(45453420 + 20*86400000, "20d12h37m", DurationFormat.COMPACT);
+ }
+
+ @Test
+ public void testVerboseManyDayHourMin() {
+ validateDurationFormat(45453420 + 20*86400000, "20 day 12 hr 37 min 33.420 sec", DurationFormat.VERBOSE);
+ }
+}
[4/4] drill git commit: DRILL-5157: Multiple Snappy versions on class
path
Posted by jn...@apache.org.
DRILL-5157: Multiple Snappy versions on class path
Multiple Snappy versions on class path; causes unit test failures.
This fix updates the Snappy library and adds dependency management to
exclude older versions brought in by Avro and Parquet.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/38f816a4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/38f816a4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/38f816a4
Branch: refs/heads/master
Commit: 38f816a45924654efd085bf7f1da7d97a4a51e38
Parents: 0b8ef9e
Author: Paul Rogers <pr...@maprtech.com>
Authored: Tue Dec 27 17:21:09 2016 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Tue Feb 21 00:53:07 2017 -0800
----------------------------------------------------------------------
exec/java-exec/pom.xml | 11 ++++++-----
pom.xml | 27 +++++++++++++++++++++++++++
2 files changed, 33 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/38f816a4/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 000d447..2acda3a 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -376,11 +376,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- <version>1.0.5-M3</version>
- </dependency>
- <dependency>
<groupId>com.carrotsearch</groupId>
<artifactId>hppc</artifactId>
<version>0.7.1</version>
@@ -429,6 +424,12 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.7.7</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/38f816a4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6813a72..0f7b54e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -609,6 +609,11 @@
<version>0.9.44</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>1.1.2.6</version>
+ </dependency>
</dependencies>
@@ -1331,6 +1336,17 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <version>${parquet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
@@ -1772,6 +1788,17 @@
<artifactId>sqlline</artifactId>
<version>${sqlline.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <version>${parquet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>org.apache.hadoop</groupId>