You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/18 06:22:11 UTC
[01/13] git commit: DRILL-1002: Fix
TestSqlBracketlessSyntax.checkComplexExpressionParsing() on Windows
Repository: incubator-drill
Updated Branches:
refs/heads/master f86639c31 -> 894037ab6
DRILL-1002: Fix TestSqlBracketlessSyntax.checkComplexExpressionParsing() on Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4d17aeab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4d17aeab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4d17aeab
Branch: refs/heads/master
Commit: 4d17aeab866a50bfe3a5d43b3fc874ae2ca84c8a
Parents: f86639c
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Mon Jun 16 23:18:02 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon Jun 16 23:25:02 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/test/DrillAssert.java | 60 ++++++++++++++++++++
.../exec/sql/TestSqlBracketlessSyntax.java | 4 +-
.../org/apache/drill/jdbc/test/TestViews.java | 10 ++--
3 files changed, 67 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4d17aeab/common/src/test/java/org/apache/drill/test/DrillAssert.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/test/DrillAssert.java b/common/src/test/java/org/apache/drill/test/DrillAssert.java
new file mode 100644
index 0000000..ddd3b93
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/test/DrillAssert.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import static org.junit.Assert.fail;
+
+public class DrillAssert {
+
+ public static void assertMultiLineStringEquals(String expected, String actual) {
+ assertMultiLineStringEquals(null, expected, actual);
+ }
+
+ public static void assertMultiLineStringEquals(String message, String expected, String actual) {
+ outside:
+ if (expected == actual) {
+ return;
+ } else if (expected != null && actual != null) {
+ int idx1 = 0, idx2 = 0;
+ char ch1, ch2;
+ while (idx1 < expected.length() && idx2 < actual.length()) {
+ ch1 = expected.charAt(idx1);
+ ch2 = actual.charAt(idx2);
+ if (isNewLineChar(ch1)) {
+ idx1++; continue;
+ } else if (isNewLineChar(ch2)) {
+ idx2++; continue;
+ } else if (ch1 != ch2) {
+ break outside;
+ } else {
+ idx1++; idx2++;
+ }
+ }
+ // skip newlines at the end
+ while(idx1 < expected.length() && isNewLineChar(expected.charAt(idx1))) idx1++;
+ while(idx2 < actual.length() && isNewLineChar(actual.charAt(idx2))) idx2++;
+ if (idx1 == expected.length() && idx2 == actual.length()) return;
+ }
+
+ fail(message != null ? message : "Expected: " + expected + ", but was: " + actual);
+ }
+
+ private static boolean isNewLineChar(char ch) {
+ return (ch == '\r' || ch == '\n');
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4d17aeab/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
index 1204e7c..ca1efab 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
@@ -25,8 +25,8 @@ import net.hydromatic.optiq.tools.StdFrameworkConfig;
import org.apache.drill.exec.planner.sql.DrillConvertletTable;
import org.apache.drill.exec.planner.sql.parser.CompoundIdentifierConverter;
import org.apache.drill.exec.planner.sql.parser.impl.DrillParserImpl;
+import org.apache.drill.test.DrillAssert;
import org.eigenbase.sql.SqlNode;
-import org.junit.Assert;
import org.junit.Test;
public class TestSqlBracketlessSyntax {
@@ -61,7 +61,7 @@ public class TestSqlBracketlessSyntax {
SqlNode rewritten = node.accept(new CompoundIdentifierConverter());
String rewrittenQuery = rewritten.toString();
- Assert.assertEquals(expected, rewrittenQuery);
+ DrillAssert.assertMultiLineStringEquals(expected, rewrittenQuery);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4d17aeab/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 6f595af..8e7131c 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -18,8 +18,10 @@
package org.apache.drill.jdbc.test;
import com.google.common.base.Function;
+
import org.apache.commons.io.FileUtils;
import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
+import org.apache.drill.test.DrillAssert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -34,8 +36,6 @@ import static org.junit.Assert.assertTrue;
/** Contains tests for creating/droping and using views in Drill. */
public class TestViews extends JdbcTestQueryBase {
- private final static String NEW_LINE = System.getProperty("line.separator");
-
@BeforeClass
public static void generateHive() throws Exception{
new HiveTestDataGenerator().generateTestData();
@@ -283,9 +283,9 @@ public class TestViews extends JdbcTestQueryBase {
"WHERE TABLE_NAME = 'testview3'");
result = JdbcAssert.toString(resultSet).trim();
resultSet.close();
- expected = "TABLE_CATALOG=DRILL; TABLE_SCHEMA=dfs.tmp; TABLE_NAME=testview3; VIEW_DEFINITION=SELECT *"+NEW_LINE+"FROM `hive`.`kv`";
- assertTrue(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected),
- expected.equals(result));
+ expected = "TABLE_CATALOG=DRILL; TABLE_SCHEMA=dfs.tmp; TABLE_NAME=testview3; VIEW_DEFINITION=SELECT *\nFROM `hive`.`kv`";
+ DrillAssert.assertMultiLineStringEquals(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected),
+ expected, result);
// test record in INFORMATION_SCHEMA.TABLES
resultSet = statement.executeQuery("SELECT * FROM INFORMATION_SCHEMA.`TABLES` " +
[12/13] git commit: Enhancements to capture HashAggregate and
HashJoin runtime metrics.
Posted by ja...@apache.org.
Enhancements to capture HashAggregate and HashJoin runtime metrics.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/28992889
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/28992889
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/28992889
Branch: refs/heads/master
Commit: 28992889090c000104bd549230f8a7fe4ee0558c
Parents: e170cf0
Author: Aman Sinha <as...@maprtech.com>
Authored: Sat Jun 14 23:40:57 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:31:27 2014 -0700
----------------------------------------------------------------------
.../physical/impl/aggregate/HashAggBatch.java | 3 +-
.../impl/aggregate/HashAggTemplate.java | 21 +++++++++++--
.../physical/impl/aggregate/HashAggregator.java | 4 ++-
.../exec/physical/impl/common/HashTable.java | 2 ++
.../physical/impl/common/HashTableMetrics.java | 33 ++++++++++++++++++++
.../physical/impl/common/HashTableStats.java | 30 ++++++++++++++++++
.../physical/impl/common/HashTableTemplate.java | 15 +++++++++
.../exec/physical/impl/join/HashJoinBatch.java | 16 +++++++++-
.../org/apache/drill/TestExampleQueries.java | 2 +-
9 files changed, 120 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index ad929a4..8250682 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -220,7 +220,8 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
container.buildSchema(SelectionVectorMode.NONE);
HashAggregator agg = context.getImplementationClass(top);
- agg.setup(popConfig, context, oContext.getAllocator(), incoming, this,
+ agg.setup(popConfig, context, this.stats,
+ oContext.getAllocator(), incoming, this,
aggrExprs,
cgInner.getWorkspaceTypes(),
groupByOutFieldIds,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index f73d46c..72095b7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.physical.impl.aggregate;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import javax.inject.Named;
@@ -37,10 +36,13 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.IntHolder;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.physical.config.HashAggregate;
import org.apache.drill.exec.physical.impl.common.ChainedHashTable;
import org.apache.drill.exec.physical.impl.common.HashTable;
import org.apache.drill.exec.physical.impl.common.HashTableConfig;
+import org.apache.drill.exec.physical.impl.common.HashTableMetrics;
+import org.apache.drill.exec.physical.impl.common.HashTableStats;
import org.apache.drill.exec.physical.impl.common.HashTableTemplate.BatchHolder;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
@@ -95,6 +97,9 @@ public abstract class HashAggTemplate implements HashAggregator {
private boolean allFlushed = false;
private boolean buildComplete = false;
+ private OperatorStats stats = null;
+ private HashTableStats htStats = new HashTableStats();
+
public class BatchHolder {
private VectorContainer aggrValuesContainer; // container for aggr values (workspace variables)
@@ -166,7 +171,9 @@ public abstract class HashAggTemplate implements HashAggregator {
@Override
- public void setup(HashAggregate hashAggrConfig, FragmentContext context, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
+ public void setup(HashAggregate hashAggrConfig, FragmentContext context,
+ OperatorStats stats,
+ BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
LogicalExpression[] valueExprs,
List<TypedFieldId> valueFieldIds,
TypedFieldId[] groupByOutFieldIds,
@@ -181,6 +188,7 @@ public abstract class HashAggTemplate implements HashAggregator {
}
this.context = context;
+ this.stats = stats;
this.allocator = allocator;
this.incoming = incoming;
this.schema = incoming.getSchema();
@@ -276,6 +284,8 @@ public abstract class HashAggTemplate implements HashAggregator {
buildComplete = true;
+ updateStats(htable);
+
// output the first batch; remaining batches will be output
// in response to each next() call by a downstream operator
@@ -534,6 +544,13 @@ public abstract class HashAggTemplate implements HashAggregator {
return false;
}
+
+ private void updateStats(HashTable htable) {
+ htable.getStats(htStats);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_BUCKETS, htStats.numBuckets);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_ENTRIES, htStats.numEntries);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_RESIZING, htStats.numResizing);
+ }
// Code-generated methods (implemented in HashAggBatch)
public abstract void doSetup(@Named("incoming") RecordBatch incoming);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
index 641d377..d14880c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.physical.config.HashAggregate;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
@@ -41,7 +42,8 @@ public interface HashAggregator {
RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR
}
- public abstract void setup(HashAggregate hashAggrConfig, FragmentContext context, BufferAllocator allocator, RecordBatch incoming,
+ public abstract void setup(HashAggregate hashAggrConfig, FragmentContext context,
+ OperatorStats stats, BufferAllocator allocator, RecordBatch incoming,
HashAggBatch outgoing, LogicalExpression[] valueExprs,
List<TypedFieldId> valueFieldIds,
TypedFieldId[] keyFieldIds,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
index 46cb47d..429ec63 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -52,6 +52,8 @@ public interface HashTable {
public int containsKey(int incomingRowIdx, boolean isProbe);
+ public void getStats(HashTableStats stats);
+
public int size();
public boolean isEmpty();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableMetrics.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableMetrics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableMetrics.java
new file mode 100644
index 0000000..ee84855
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableMetrics.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+import org.apache.drill.exec.ops.MetricDef;
+
+public enum HashTableMetrics implements MetricDef {
+
+ HTABLE_NUM_BUCKETS,
+ HTABLE_NUM_ENTRIES,
+ HTABLE_NUM_RESIZING;
+
+ @Override
+ public int metricId() {
+ return ordinal();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableStats.java
new file mode 100644
index 0000000..848d860
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableStats.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+
+public class HashTableStats {
+ public int numBuckets;
+ public int numEntries;
+ public int numResizing;
+
+ public HashTableStats() {
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index a8af5ea..f7cadf1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -88,6 +88,8 @@ public abstract class HashTableTemplate implements HashTable {
private MaterializedField dummyIntField;
+ private int numResizing = 0;
+
// This class encapsulates the links, keys and values for up to BATCH_SIZE
// *unique* records. Thus, suppose there are N incoming record batches, each
// of size BATCH_SIZE..but they have M unique keys altogether, the number of
@@ -363,10 +365,21 @@ public abstract class HashTableTemplate implements HashTable {
return startIndices.getAccessor().getValueCount();
}
+ public int numResizing() {
+ return numResizing;
+ }
+
public int size() {
return numEntries;
}
+ public void getStats(HashTableStats stats) {
+ assert stats != null;
+ stats.numBuckets = numBuckets();
+ stats.numEntries = numEntries;
+ stats.numResizing = numResizing;
+ }
+
public boolean isEmpty() {
return numEntries == 0;
}
@@ -594,6 +607,8 @@ public abstract class HashTableTemplate implements HashTable {
bh.dump(idx);
}
}
+
+ numResizing++;
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index ddc31ee..9343912 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -41,6 +41,8 @@ import org.apache.drill.exec.physical.config.HashJoinPOP;
import org.apache.drill.exec.physical.impl.common.ChainedHashTable;
import org.apache.drill.exec.physical.impl.common.HashTable;
import org.apache.drill.exec.physical.impl.common.HashTableConfig;
+import org.apache.drill.exec.physical.impl.common.HashTableMetrics;
+import org.apache.drill.exec.physical.impl.common.HashTableStats;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch;
import org.apache.drill.exec.record.AbstractRecordBatch;
@@ -132,6 +134,8 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
boolean firstOutputBatch = true;
IterOutcome leftUpstream = IterOutcome.NONE;
+
+ private HashTableStats htStats = new HashTableStats();
@Override
public int getRecordCount() {
@@ -161,10 +165,13 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
// Build the hash table, using the build side record batches.
executeBuildPhase();
+ // Update the hash table related stats for the operator
+ updateStats(this.hashTable);
+
// Create the run time generated code needed to probe and project
hashJoinProbe = setupHashJoinProbe();
}
-
+
// Store the number of records projected
if (hashTable != null
|| joinType != JoinRelType.INNER) {
@@ -432,6 +439,13 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
this.conditions = popConfig.getConditions();
}
+ private void updateStats(HashTable htable) {
+ htable.getStats(htStats);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_BUCKETS, htStats.numBuckets);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_ENTRIES, htStats.numEntries);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_RESIZING, htStats.numResizing);
+ }
+
@Override
public void killIncoming() {
this.left.kill();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/28992889/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 59556b3..1d6ca33 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -202,5 +202,5 @@ public class TestExampleQueries extends BaseTestQuery{
test("select r_name from cp.`tpch/region.parquet` order by r_name, r_regionkey");
test("select cast(r_name as varchar(20)) from cp.`tpch/region.parquet` order by r_name");
}
-
+
}
[07/13] git commit: DRILL-995: Removed inner form (which is not
allowed). Fixed NPE and URLs.
Posted by ja...@apache.org.
DRILL-995: Removed inner form (which is not allowed). Fixed NPE and URLs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e2f57b46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e2f57b46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e2f57b46
Branch: refs/heads/master
Commit: e2f57b46b97b1831c21dbfa07fd831abedf2020e
Parents: 219e4fa
Author: Sudheesh Katkam <sk...@maprtech.com>
Authored: Mon Jun 16 14:42:02 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:04:33 2014 -0700
----------------------------------------------------------------------
.../exec/server/rest/StorageResources.java | 27 ++++++++++----------
.../src/main/resources/rest/storage/list.ftl | 6 ++---
.../src/main/resources/rest/storage/update.ftl | 13 +++-------
3 files changed, 20 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e2f57b46/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
index aa090cf..17ea72c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
@@ -86,12 +86,12 @@ public class StorageResources {
map.put("config", conf);
map.put("name", name);
map.put("exists", config != null);
- map.put("enabled", config.isEnabled());
+ map.put("enabled", config != null && config.isEnabled());
return new Viewable("/rest/storage/update.ftl", map);
}
@GET
- @Path("/{name}/enable/{val}")
+ @Path("/{name}/config/enable/{val}")
@Produces(MediaType.TEXT_HTML)
public Response setEnable(@Context UriInfo uriInfo, @PathParam("name") String name, @PathParam("val") Boolean enable) throws ExecutionSetupException {
StoragePluginConfig config = findConfig(name);
@@ -105,6 +105,14 @@ public class StorageResources {
}
@GET
+ @Path("/{name}/config/delete")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable deleteConfig(@PathParam("name") String name) {
+ storage.deletePlugin(name);
+ return new Viewable("/rest/status.ftl", "Deleted " + name);
+ }
+
+ @GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{name}/config")
public StoragePluginConfig getConfig(@PathParam("name") String name) {
@@ -126,22 +134,13 @@ public class StorageResources {
@POST
@Path("/config/update")
- @Produces(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.TEXT_HTML)
@Consumes("application/x-www-form-urlencoded")
- public JsonResult createTrackInJSON(@FormParam("name") String name, @FormParam("config") String storagePluginConfig)
+ public Viewable createTrackInJSON(@FormParam("name") String name, @FormParam("config") String storagePluginConfig)
throws ExecutionSetupException, JsonParseException, JsonMappingException, IOException {
StoragePluginConfig config = mapper.readValue(new StringReader(storagePluginConfig), StoragePluginConfig.class);
storage.createOrUpdate(name, config, true);
- return r("success");
- }
-
- @POST
- @Path("/config/delete")
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes("application/x-www-form-urlencoded")
- public JsonResult deleteConfig(@FormParam("name") String name) {
- storage.deletePlugin(name);
- return r("success");
+ return new Viewable("/rest/status.ftl", "Updated " + name);
}
private JsonResult r(String message) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e2f57b46/exec/java-exec/src/main/resources/rest/storage/list.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/storage/list.ftl b/exec/java-exec/src/main/resources/rest/storage/list.ftl
index 5e50616..3636fbb 100644
--- a/exec/java-exec/src/main/resources/rest/storage/list.ftl
+++ b/exec/java-exec/src/main/resources/rest/storage/list.ftl
@@ -29,9 +29,9 @@
<td style="border:none;">
<a class="btn btn-primary" href="/storage/${plugin.name}/config/update">Update</a>
<#if plugin.enabled>
- <a class="btn btn-default" href="/storage/${plugin.name}/enable/false">Disable</a>
+ <a class="btn btn-default" href="/storage/${plugin.name}/config/enable/false">Disable</a>
<#else>
- <a class="btn btn-primary" href="/storage/${plugin.name}/enable/true">Enable</a>
+ <a class="btn btn-primary" href="/storage/${plugin.name}/config/enable/true">Enable</a>
</#if>
</td>
</tr>
@@ -49,7 +49,7 @@
function doSubmit() {
var name = document.getElementById("storageName");
var form = document.getElementById("newStorage");
- form.action = "/storage/" + name.value + "/config/update?";
+ form.action = "/storage/" + name.value + "/config/update";
form.submit();
}
</script>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e2f57b46/exec/java-exec/src/main/resources/rest/storage/update.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/storage/update.ftl b/exec/java-exec/src/main/resources/rest/storage/update.ftl
index ae60852..f4b9e12 100644
--- a/exec/java-exec/src/main/resources/rest/storage/update.ftl
+++ b/exec/java-exec/src/main/resources/rest/storage/update.ftl
@@ -28,17 +28,12 @@
<#if model.exists >Update<#else>Create</#if>
</button>
<#if model.enabled>
- <a class="btn btn-default" href="/storage/${model.name}/enable/false">Disable</a>
+ <a class="btn btn-default" href="/storage/${model.name}/config/enable/false">Disable</a>
<#else>
- <a class="btn btn-primary" href="/storage/${model.name}/enable/true">Enable</a>
+ <a class="btn btn-primary" href="/storage/${model.name}/config/enable/true">Enable</a>
</#if>
- <#if model.exists >
- <form role="form" action="/storage/config/delete" method="POST">
- <input type="hidden" name="name" value="${model.name}" />
- <button type="submit" class="btn btn-default" onclick="return confirm('Are you sure?')">
- Delete
- </button>
- </form>
+ <#if model.exists>
+ <a class="btn btn-danger" href="/storage/${model.name}/config/delete">Delete</a>
</#if>
</form>
</#macro>
[04/13] git commit: DRILL-793: Fix output type's scale and precision
for math functions.
Posted by ja...@apache.org.
DRILL-793: Fix output type's scale and precision for math functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/1465e11c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/1465e11c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/1465e11c
Branch: refs/heads/master
Commit: 1465e11ca58fc139ae2f444b236c2db514276687
Parents: 5079f8b
Author: Mehant Baid <me...@gmail.com>
Authored: Thu Jun 12 18:09:57 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:04:09 2014 -0700
----------------------------------------------------------------------
.../DecimalScalePrecisionDivideFunction.java | 65 +++++++++++++
.../util/DecimalScalePrecisionMulFunction.java | 52 +++++++++++
.../drill/common/util/DecimalUtility.java | 38 ++++++++
.../templates/Decimal/DecimalFunctions.java | 97 +++++++++++++++-----
.../expr/fn/DrillDecimalDivScaleFuncHolder.java | 28 ++++--
.../expr/fn/DrillDecimalSumScaleFuncHolder.java | 25 ++++-
.../drill/exec/expr/fn/DrillFuncHolder.java | 2 +-
.../drill/exec/resolver/TypeCastRules.java | 15 +--
.../drill/exec/physical/impl/TestDecimal.java | 5 +-
.../resources/decimal/test_decimal_complex.json | 8 +-
.../drill/jdbc/test/TestFunctionsQuery.java | 15 +++
11 files changed, 299 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
new file mode 100644
index 0000000..5a53603
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.common.util;
+
+/*
+ * Here we compute the scale and precision of the output decimal data type
+ * based on the input scale and precision. Since division operation can be
+ * a multiplication operation we compute the scale to be the sum of the inputs.
+ * The precision is computed by getting the sum of integer digits of the input
+ * and adding it with scale. The scale is further expanded to occupy the remaining
+ * digits in the given precision range
+ *
+ * Eg: Input1 : precision = 5, scale = 3 ==> max integer digits = 2
+ * Input2 : precision = 7, scale = 4 ==> max integer digits = 3
+ *
+ * Output: max integer digits ==> 2 + 3 = 5
+ * max scale ==> 3 + 4 = 7
+ *
+ * Minimum precision required ==> 5 + 7 = 12
+ *
+ * Since our minimum precision required is 12, we will use DECIMAL18 as the output type
+ * but since this is divide we will grant the remaining digits in DECIMAL18 to scale
+ * so we have the following
+ * output scale ==> 7 + (18 - 12) = 13
+ * output precision ==> 18
+ */
+public class DecimalScalePrecisionDivideFunction {
+ private int outputScale = 0;
+ private int outputPrecision = 0;
+
+ public DecimalScalePrecisionDivideFunction(int leftPrecision, int leftScale, int rightPrecision, int rightScale) {
+ // compute the output scale and precision here
+ outputScale = leftScale + rightScale;
+ int integerDigits = (leftPrecision - leftScale) + (rightPrecision - rightScale);
+
+ outputPrecision = DecimalUtility.getPrecisionRange(outputScale + integerDigits);
+
+ // Try and increase the scale if we have any room
+ outputScale = (outputPrecision - integerDigits >= 0) ? (outputPrecision - integerDigits) : 0;
+ }
+
+ public int getOutputScale() {
+ return outputScale;
+ }
+
+ public int getOutputPrecision() {
+ return outputPrecision;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionMulFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionMulFunction.java b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionMulFunction.java
new file mode 100644
index 0000000..1fd3427
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionMulFunction.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.common.util;
+
+/*
+ * Here we compute the output scale and precision of the multiply function.
+ * We simply add the input scale and precision to determine the output's scale
+ * and precision
+ */
+public class DecimalScalePrecisionMulFunction {
+ private int outputScale = 0;
+ private int outputPrecision = 0;
+
+ public DecimalScalePrecisionMulFunction(int leftPrecision, int leftScale, int rightPrecision, int rightScale) {
+ // compute the output scale and precision here
+ outputScale = leftScale + rightScale;
+ int integerDigits = (leftPrecision - leftScale) + (rightPrecision - rightScale);
+
+ outputPrecision = integerDigits + outputScale;
+
+ // If we are beyond the maximum precision range, cut down the fractional part
+ if (outputPrecision > 38) {
+ outputPrecision = 38;
+ outputScale = (outputPrecision - integerDigits >= 0) ? (outputPrecision - integerDigits) : 0;
+ }
+ }
+
+ public int getOutputScale() {
+ return outputScale;
+ }
+
+ public int getOutputPrecision() {
+ return outputPrecision;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
index 4cc80ea..7f1a4a0 100644
--- a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
@@ -19,6 +19,7 @@ package org.apache.drill.common.util;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import org.apache.drill.common.types.TypeProtos;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -646,5 +647,42 @@ public class DecimalUtility {
}
}
+ /*
+ * Function returns the Minor decimal type given the precision
+ */
+ public static TypeProtos.MinorType getDecimalDataType(int precision) {
+ if (precision <= 9) {
+ return TypeProtos.MinorType.DECIMAL9;
+ } else if (precision <= 18) {
+ return TypeProtos.MinorType.DECIMAL18;
+ } else if (precision <= 28) {
+ return TypeProtos.MinorType.DECIMAL28SPARSE;
+ } else {
+ return TypeProtos.MinorType.DECIMAL38SPARSE;
+ }
+ }
+
+ public static int getMaxPrecision(TypeProtos.MinorType decimalType) {
+ if (decimalType == TypeProtos.MinorType.DECIMAL9) {
+ return 9;
+ } else if (decimalType == TypeProtos.MinorType.DECIMAL18) {
+ return 18;
+ } else if (decimalType == TypeProtos.MinorType.DECIMAL28SPARSE) {
+ return 28;
+ } else if (decimalType == TypeProtos.MinorType.DECIMAL38SPARSE) {
+ return 38;
+ }
+ return 0;
+ }
+
+
+ /*
+ * Given a precision it provides the max precision of that decimal data type;
+ * For eg: given the precision 12, we would use DECIMAL18 to store the data
+ * which has a max precision range of 18 digits
+ */
+ public static int getPrecisionRange(int precision) {
+ return getMaxPrecision(getDecimalDataType(precision));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
index b294396..a41fb20 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
@@ -379,6 +379,8 @@ public class ${type.name}Functions {
@Param ${type.name}Holder right;
@Workspace ByteBuf buffer;
@Workspace int[] tempResult;
+ @Workspace int outputScale;
+ @Workspace int outputPrecision;
@Output ${type.name}Holder result;
public void setup(RecordBatch incoming) {
@@ -386,10 +388,20 @@ public class ${type.name}Functions {
buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
buffer = new io.netty.buffer.SwappedByteBuf(buffer);
tempResult = new int[${type.storage} * ${type.storage}];
+ outputPrecision = Integer.MIN_VALUE;
}
public void eval() {
+ if (outputPrecision == Integer.MIN_VALUE) {
+ org.apache.drill.common.util.DecimalScalePrecisionMulFunction resultScalePrec =
+ new org.apache.drill.common.util.DecimalScalePrecisionMulFunction((int) left.precision, (int) left.scale, (int) right.precision, (int) right.scale);
+ outputScale = resultScalePrec.getOutputScale();
+ outputPrecision = resultScalePrec.getOutputPrecision();
+ }
+ // Set the scale and precision
+ result.scale = outputScale;
+ result.precision = outputPrecision;
result.buffer = buffer;
result.start = 0;
@@ -474,10 +486,6 @@ public class ${type.name}Functions {
result.setInteger(outputIndex--, 0);
}
- // Set the scale and precision
- result.scale = left.scale + right.scale;
- result.precision = result.maxPrecision;
-
result.sign = (left.sign == right.sign) ? false : true;
}
}
@@ -489,17 +497,25 @@ public class ${type.name}Functions {
@Param ${type.name}Holder right;
@Output ${type.name}Holder result;
@Workspace ByteBuf buffer;
+ @Workspace int outputScale;
+ @Workspace int outputPrecision;
public void setup(RecordBatch incoming) {
int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ outputPrecision = Integer.MIN_VALUE;
}
public void eval() {
-
- result.scale = left.scale;
- result.precision = left.precision;
+ if (outputPrecision == Integer.MIN_VALUE) {
+ org.apache.drill.common.util.DecimalScalePrecisionDivideFunction resultScalePrec =
+ new org.apache.drill.common.util.DecimalScalePrecisionDivideFunction((int) left.precision, (int) left.scale, (int) right.precision, (int) right.scale);
+ outputScale = resultScalePrec.getOutputScale();
+ outputPrecision = resultScalePrec.getOutputPrecision();
+ }
+ result.scale = outputScale;
+ result.precision = outputPrecision;
result.buffer = buffer;
result.start = 0;
@@ -524,17 +540,25 @@ public class ${type.name}Functions {
@Param ${type.name}Holder right;
@Output ${type.name}Holder result;
@Workspace ByteBuf buffer;
+ @Workspace int outputScale;
+ @Workspace int outputPrecision;
public void setup(RecordBatch incoming) {
int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ outputPrecision = Integer.MIN_VALUE;
}
public void eval() {
-
- result.scale = left.scale;
- result.precision = left.precision;
+ if (outputPrecision == Integer.MIN_VALUE) {
+ org.apache.drill.common.util.DecimalScalePrecisionDivideFunction resultScalePrec =
+ new org.apache.drill.common.util.DecimalScalePrecisionDivideFunction((int) left.precision, (int) left.scale, (int) right.precision, (int) right.scale);
+ outputScale = resultScalePrec.getOutputScale();
+ outputPrecision = resultScalePrec.getOutputPrecision();
+ }
+ result.scale = outputScale;
+ result.precision = outputPrecision;
result.buffer = buffer;
result.start = 0;
@@ -542,7 +566,7 @@ public class ${type.name}Functions {
java.math.BigDecimal denominator = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(right.buffer, right.start, right.nDecimalDigits, right.scale, true);
java.math.BigDecimal output = numerator.remainder(denominator);
- output.setScale(left.scale, java.math.BigDecimal.ROUND_DOWN);
+ output.setScale(result.scale, java.math.BigDecimal.ROUND_DOWN);
// Initialize the result buffer
for (int i = 0; i < ${type.storage}; i++) {
@@ -1204,15 +1228,24 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
+ @Workspace int outputScale;
+ @Workspace int outputPrecision;
@Output ${type.name}Holder result;
- public void setup(RecordBatch incoming) {}
+ public void setup(RecordBatch incoming) {
+ outputPrecision = Integer.MIN_VALUE;
+ }
public void eval() {
-
+ if (outputPrecision == Integer.MIN_VALUE) {
+ org.apache.drill.common.util.DecimalScalePrecisionMulFunction resultScalePrec =
+ new org.apache.drill.common.util.DecimalScalePrecisionMulFunction((int) left.precision, (int) left.scale, (int) right.precision, (int) right.scale);
+ outputScale = resultScalePrec.getOutputScale();
+ outputPrecision = resultScalePrec.getOutputPrecision();
+ }
result.value = left.value * right.value;
- result.precision = result.maxPrecision;
- result.scale = left.scale + right.scale;
+ result.precision = outputPrecision;
+ result.scale = outputScale;
}
}
@@ -1242,12 +1275,23 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
@Output ${type.name}Holder result;
+ @Workspace int outputScale;
+ @Workspace int outputPrecision;
- public void setup(RecordBatch incoming) {}
+ public void setup(RecordBatch incoming) {
+ outputPrecision = Integer.MIN_VALUE;
+ }
public void eval() {
- result.scale = left.scale;
+ if (outputPrecision == Integer.MIN_VALUE) {
+ org.apache.drill.common.util.DecimalScalePrecisionDivideFunction resultScalePrec =
+ new org.apache.drill.common.util.DecimalScalePrecisionDivideFunction((int) left.precision, (int) left.scale, (int) right.precision, (int) right.scale);
+ outputScale = resultScalePrec.getOutputScale();
+ outputPrecision = resultScalePrec.getOutputPrecision();
+ }
+ result.scale = outputScale;
+ result.precision = outputPrecision;
java.math.BigDecimal numerator = new java.math.BigDecimal(java.math.BigInteger.valueOf(left.value), left.scale);
java.math.BigDecimal denominator = new java.math.BigDecimal(java.math.BigInteger.valueOf(right.value), right.scale);
@@ -1263,21 +1307,30 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
+ @Workspace int outputScale;
+ @Workspace int outputPrecision;
@Output ${type.name}Holder result;
- public void setup(RecordBatch incoming) {}
+ public void setup(RecordBatch incoming) {
+ outputPrecision = Integer.MIN_VALUE;
+ }
public void eval() {
-
+ if (outputPrecision == Integer.MIN_VALUE) {
+ org.apache.drill.common.util.DecimalScalePrecisionDivideFunction resultScalePrec =
+ new org.apache.drill.common.util.DecimalScalePrecisionDivideFunction((int) left.precision, (int) left.scale, (int) right.precision, (int) right.scale);
+ outputScale = resultScalePrec.getOutputScale();
+ outputPrecision = resultScalePrec.getOutputPrecision();
+ }
+ result.precision = outputPrecision;
+ result.scale = outputScale;
java.math.BigDecimal numerator = new java.math.BigDecimal(java.math.BigInteger.valueOf(left.value), left.scale);
java.math.BigDecimal denominator = new java.math.BigDecimal(java.math.BigInteger.valueOf(right.value), right.scale);
java.math.BigDecimal output = numerator.remainder(denominator);
- output.setScale(left.scale, java.math.BigDecimal.ROUND_DOWN);
+ output.setScale(result.scale, java.math.BigDecimal.ROUND_DOWN);
result.value = output.unscaledValue().${type.storage}Value();
- result.precision = result.maxPrecision;
- result.scale = left.scale;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
index af239e4..941dc49 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
@@ -24,6 +24,8 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.util.DecimalScalePrecisionDivideFunction;
+import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -36,12 +38,15 @@ public class DrillDecimalDivScaleFuncHolder extends DrillSimpleFuncHolder{
super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports);
}
+ /*
+ * This function scope is used by divide functions for decimal data type.
+ * DecimalScalePrecisionDivideFunction is used to compute the output types'
+ * scale and precision
+ */
@Override
public MajorType getReturnType(List<LogicalExpression> args) {
TypeProtos.DataMode mode = returnValue.type.getMode();
- int scale = 0;
- int precision = 0;
if (nullHandling == NullHandling.NULL_IF_NULL) {
// if any one of the input types is nullable, then return nullable return type
@@ -53,12 +58,21 @@ public class DrillDecimalDivScaleFuncHolder extends DrillSimpleFuncHolder{
}
}
- /* Set the scale to be the same as the fist input's scale
- * Used by divide and modulo functions
+
+ /* Get the result's scale and precision. This is a function scope for Divide function, assert we have
+ * only two inputs
*/
- scale = args.get(0).getMajorType().getScale();
- precision = args.get(0).getMajorType().getPrecision();
+ assert args.size() == 2;
- return (TypeProtos.MajorType.newBuilder().setMinorType(returnValue.type.getMinorType()).setScale(scale).setPrecision(precision).setMode(mode).build());
+ DecimalScalePrecisionDivideFunction outputScalePrec =
+ new DecimalScalePrecisionDivideFunction(args.get(0).getMajorType().getPrecision(), args.get(0).getMajorType().getScale(),
+ args.get(1).getMajorType().getPrecision(), args.get(1).getMajorType().getScale());
+ return (TypeProtos.MajorType.newBuilder().setMinorType(DecimalUtility.getDecimalDataType(outputScalePrec.getOutputPrecision()))
+ .setScale(outputScalePrec.getOutputScale()).setPrecision(outputScalePrec.getOutputPrecision()).setMode(mode).build());
+ }
+
+ @Override
+ public boolean checkPrecisionRange() {
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
index 2e82966..6c04afc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
@@ -24,6 +24,8 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.util.DecimalScalePrecisionMulFunction;
+import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -40,19 +42,32 @@ public class DrillDecimalSumScaleFuncHolder extends DrillSimpleFuncHolder{
public MajorType getReturnType(List<LogicalExpression> args) {
TypeProtos.DataMode mode = returnValue.type.getMode();
- int scale = 0;
- int precision = 0;
if (nullHandling == NullHandling.NULL_IF_NULL) {
// if any one of the input types is nullable, then return nullable return type
for (LogicalExpression e : args) {
if (e.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) {
mode = TypeProtos.DataMode.OPTIONAL;
+ break;
}
- scale += e.getMajorType().getScale();
- precision = Math.max(precision, e.getMajorType().getPrecision());
}
}
- return (TypeProtos.MajorType.newBuilder().setMinorType(returnValue.type.getMinorType()).setScale(scale).setPrecision(precision).setMode(mode).build());
+
+ /* Get the result's scale and precision. This is a function scope for Multiply function, assert we have
+ * only two inputs
+ */
+ assert args.size() == 2;
+
+ DecimalScalePrecisionMulFunction outputScalePrec =
+ new DecimalScalePrecisionMulFunction(args.get(0).getMajorType().getPrecision(), args.get(0).getMajorType().getScale(),
+ args.get(1).getMajorType().getPrecision(), args.get(1).getMajorType().getScale());
+ return (TypeProtos.MajorType.newBuilder().setMinorType(DecimalUtility.getDecimalDataType(outputScalePrec.getOutputPrecision()))
+ .setScale(outputScalePrec.getOutputScale()).setPrecision(outputScalePrec.getOutputPrecision()).setMode(mode).build());
+ }
+
+ @Override
+ public boolean checkPrecisionRange() {
+ return true;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index fd687af..fc8dc00 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -294,7 +294,7 @@ public abstract class DrillFuncHolder {
}
- public boolean matchInputOutputType() {
+ public boolean checkPrecisionRange() {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index 854342c..bf202c8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
@@ -794,13 +795,13 @@ public class TypeCastRules {
// number of arguments that could implicitly casts using precedence map or didn't require casting at all
int nCasts = 0;
- // Check if the function holder requires the input type and output type to match
- if (holder.matchInputOutputType() == true) {
- MinorType outputType = holder.getReturnType(call.args).getMinorType();
- for (int i = 0; i < holder.getParamCount(); i++) {
- if (call.args.get(i).getMajorType().getMinorType() != outputType) {
- return -1;
- }
+ /*
+ * If we are determining function holder for decimal data type, we need to make sure the output type of
+ * the function can fit the precision that we need based on the input types.
+ */
+ if (holder.checkPrecisionRange() == true) {
+ if (DecimalUtility.getMaxPrecision(holder.getReturnType().getMinorType()) < holder.getReturnType(call.args).getPrecision()) {
+ return -1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
index 2df4d18..f485378 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
@@ -34,6 +34,7 @@ import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.ValueVector;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.base.Charsets;
@@ -208,22 +209,18 @@ public class TestDecimal extends PopUnitTestBase{
String addOutput[] = {"-99999998877.700000000", "11.423456789", "123456789.100000000", "-0.119998000", "100000000112.423456789" , "-99999999879.907000000", "123456789123456801.300000000"};
String subtractOutput[] = {"-100000001124.300000000", "10.823456789", "-123456788.900000000", "-0.120002000", "99999999889.823456789", "-100000000122.093000000", "123456789123456776.700000000"};
- String multiplyOutput[] = {"-112330000001123.300000000000000000", "3.337037036700000000" , "12345678.900000000000000000", "-0.000000240000000000" , "11130000000125.040740615700000000" , "-12109300000121.093000000000000000", "1518518506218518504.700000000000000000" };
Iterator<VectorWrapper<?>> itr = batchLoader.iterator();
ValueVector.Accessor addAccessor = itr.next().getValueVector().getAccessor();
ValueVector.Accessor subAccessor = itr.next().getValueVector().getAccessor();
- ValueVector.Accessor mulAccessor = itr.next().getValueVector().getAccessor();
for (int i = 0; i < addAccessor.getValueCount(); i++) {
assertEquals(addAccessor.getObject(i).toString(), addOutput[i]);
assertEquals(subAccessor.getObject(i).toString(), subtractOutput[i]);
- assertEquals(mulAccessor.getObject(i).toString(), multiplyOutput[i]);
}
assertEquals(7, addAccessor.getValueCount());
assertEquals(7, subAccessor.getValueCount());
- assertEquals(7, mulAccessor.getValueCount());
batchLoader.clear();
for (QueryResultBatch result : results) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/java-exec/src/test/resources/decimal/test_decimal_complex.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/decimal/test_decimal_complex.json b/exec/java-exec/src/test/resources/decimal/test_decimal_complex.json
index b2f1929..3f98174 100644
--- a/exec/java-exec/src/test/resources/decimal/test_decimal_complex.json
+++ b/exec/java-exec/src/test/resources/decimal/test_decimal_complex.json
@@ -22,7 +22,7 @@
"ref" : "DE",
"expr" : " (cast(B as decimal38sparse(38, 9))) "
},
- {"ref" : "DE1", "expr": " cast(A as decimal38sparse(38, 9))" }
+ {"ref" : "DE1", "expr": " cast(A as decimal18(15, 6))" }
],
"child" : 1
@@ -34,8 +34,7 @@
"ref" : "DEC38ADD",
"expr" : " (DE + DE1) "
},
- {"ref" : "DEC38SUB" , "expr" : " (DE - DE1) " },
- {"ref" : "DEC38MUL" , "expr" : " (DE * DE1) " }
+ {"ref" : "DEC38SUB" , "expr" : " (DE - DE1) " }
],
"child" : 2
@@ -47,8 +46,7 @@
"ref" : "DEC38ADD",
"expr" : " cast(DEC38ADD as varchar(100)) "
},
- {"ref" : "DEC38SUB" , "expr" : " cast(DEC38SUB as varchar(100)) " },
- {"ref" : "DEC38MUL" , "expr" : " cast(DEC38MUL as varchar(100)) " }
+ {"ref" : "DEC38SUB" , "expr" : " cast(DEC38SUB as varchar(100)) " }
],
"child" : 3
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1465e11c/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index 60b8f82..c2d90fe 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -523,4 +523,19 @@ public class TestFunctionsQuery {
.returns(
"DECIMAL_DOUBLE_CAST=1.0001\n");
}
+
+ @Test
+ public void testCastDecimalDivide() throws Exception {
+ String query = "select (cast('9' as decimal(9, 1)) / cast('2' as decimal(4, 1))) as DEC9_DIV, " +
+ "cast('123456789.123456789' as decimal(18, 9)) * cast('123456789.123456789' as decimal(18, 9)) as DEC18_MUL " +
+ "from cp.`employee.json` where employee_id = 1";
+
+ JdbcAssert.withNoDefaultSchema()
+ .sql(query)
+ .returns(
+ "DEC9_DIV=4.5000000; " +
+ "DEC18_MUL=15241578780673678.515622620750190521\n");
+ }
+
+
}
[06/13] git commit: DRILL-976: Fix extract second to return only
second portion of interval; exclude hours, minutes
Posted by ja...@apache.org.
DRILL-976: Fix extract second to return only second portion of interval; exclude hours, minutes
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/219e4fa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/219e4fa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/219e4fa3
Branch: refs/heads/master
Commit: 219e4fa3f2ec5346ab2480cd5fece220638e80b4
Parents: 5e357fd
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Jun 17 02:11:55 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:04:19 2014 -0700
----------------------------------------------------------------------
.../templates/DateIntervalFunctionTemplates/Extract.java | 6 ++++--
.../org/apache/drill/jdbc/test/TestFunctionsQuery.java | 10 ++++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/219e4fa3/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
index 3f0dcee..6dae417 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
@@ -103,7 +103,8 @@ public class ${className} {
int millis = in.milliSeconds % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
out.value = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
<#elseif toUnit == "Second">
- out.value = (double) in.milliSeconds / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
+ long millis = in.milliSeconds % org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis;
+ out.value = (double) millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
</#if>
<#elseif fromUnit == "IntervalDay">
<#if toUnit == "Year" || toUnit == "Month">
@@ -116,7 +117,8 @@ public class ${className} {
int millis = in.milliSeconds % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
out.value = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
<#elseif toUnit == "Second">
- out.value = (double) in.milliSeconds/ (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
+ long millis = in.milliSeconds % org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis;
+ out.value = (double) millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
</#if>
<#else> <#-- IntervalYear type -->
<#if toUnit == "Year">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/219e4fa3/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index c2d90fe..d700763 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -537,5 +537,15 @@ public class TestFunctionsQuery {
"DEC18_MUL=15241578780673678.515622620750190521\n");
}
+ @Test
+ public void testExtractSecondFromInterval() throws Exception {
+ String query = "select extract (second from interval '1 2:30:45.100' day to second) as EXT_INTDAY " +
+ "from cp.`employee.json` where employee_id = 1";
+
+ JdbcAssert.withNoDefaultSchema()
+ .sql(query)
+ .returns(
+ "EXT_INTDAY=45.1\n");
+ }
}
[13/13] git commit: Improve OperatorStats to avoid leaking state. Fix
issue where HashJoinBatch throws NPE in stats tracking if we don't have
HashTable.
Posted by ja...@apache.org.
Improve OperatorStats to avoid leaking state. Fix issue where HashJoinBatch throws NPE in stats tracking if we don't have HashTable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/894037ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/894037ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/894037ab
Branch: refs/heads/master
Commit: 894037ab693dea425e88fb3ec3aff73ea5b15eb1
Parents: 2899288
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Jun 17 19:41:15 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 20:18:37 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/ops/OperatorStats.java | 15 +++++++++------
.../drill/exec/physical/impl/WriterRecordBatch.java | 3 ++-
.../exec/physical/impl/aggregate/HashAggBatch.java | 4 ++--
.../physical/impl/aggregate/StreamingAggBatch.java | 3 ++-
.../drill/exec/physical/impl/join/HashJoinBatch.java | 13 +++++++------
.../exec/physical/impl/join/MergeJoinBatch.java | 4 ++--
.../drill/exec/record/AbstractRecordBatch.java | 10 +++++++---
7 files changed, 31 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index 7d1e9dc..dcb73c8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -66,41 +66,44 @@ public class OperatorStats {
this.schemaCountByInput = new long[inputCount];
}
+ private String assertionError(String msg){
+ return String.format("Failure while %s for operator id %d. Currently have states of processing:%s, setup:%s, waiting:%s.", msg, operatorId, inProcessing, inSetup, inWait);
+ }
public void startSetup() {
- assert !inSetup : "Failure while starting setup. Currently in setup.";
+ assert !inSetup : assertionError("starting setup");
stopProcessing();
inSetup = true;
setupMark = System.nanoTime();
}
public void stopSetup() {
- assert inSetup : "Failure while stopping setup. Not currently in setup.";
+ assert inSetup : assertionError("stopping setup");
startProcessing();
setupNanos += System.nanoTime() - setupMark;
inSetup = false;
}
public void startProcessing() {
- assert !inProcessing : "Failure while starting processing. Currently in processing.";
+ assert !inProcessing : assertionError("starting processing");
processingMark = System.nanoTime();
inProcessing = true;
}
public void stopProcessing() {
- assert inProcessing : "Failure while stopping processing. Not currently in processing.";
+ assert inProcessing : assertionError("stopping processing");
processingNanos += System.nanoTime() - processingMark;
inProcessing = false;
}
public void startWait() {
- assert !inWait : "Failure while starting waiting. Currently in waiting.";
+ assert !inWait : assertionError("starting waiting");
stopProcessing();
inWait = true;
waitMark = System.nanoTime();
}
public void stopWait() {
- assert inWait : "Failure while stopping waiting. Currently not in waiting.";
+ assert inWait : assertionError("stopping waiting");
startProcessing();
waitNanos += System.nanoTime() - waitMark;
inWait = false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
index 2dae853..43e0dd4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
@@ -156,9 +156,10 @@ public class WriterRecordBatch extends AbstractRecordBatch<Writer> {
// update the schema in RecordWriter
stats.startSetup();
recordWriter.updateSchema(incoming.getSchema());
- stats.stopSetup();
} catch(IOException ex) {
throw new RuntimeException("Failed to update schema in RecordWriter", ex);
+ } finally{
+ stats.stopSetup();
}
eventBasedRecordWriter = new EventBasedRecordWriter(incoming.getSchema(),
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index 8250682..dd58562 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -154,14 +154,14 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
try{
stats.startSetup();
this.aggregator = createAggregatorInternal();
- stats.stopSetup();
return true;
}catch(SchemaChangeException | ClassTransformationException | IOException ex){
- stats.stopSetup();
context.fail(ex);
container.clear();
incoming.kill();
return false;
+ }finally{
+ stats.stopSetup();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index 8cad91b..ec12de9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -136,13 +136,14 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
try{
stats.startSetup();
this.aggregator = createAggregatorInternal();
- stats.stopSetup();
return true;
}catch(SchemaChangeException | ClassTransformationException | IOException ex){
context.fail(ex);
container.clear();
incoming.kill();
return false;
+ }finally{
+ stats.stopSetup();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 9343912..c43b99a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -134,8 +134,8 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
boolean firstOutputBatch = true;
IterOutcome leftUpstream = IterOutcome.NONE;
-
- private HashTableStats htStats = new HashTableStats();
+
+ private final HashTableStats htStats = new HashTableStats();
@Override
public int getRecordCount() {
@@ -171,9 +171,9 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
// Create the run time generated code needed to probe and project
hashJoinProbe = setupHashJoinProbe();
}
-
+
// Store the number of records projected
- if (hashTable != null
+ if (hashTable != null
|| joinType != JoinRelType.INNER) {
// Allocate the memory for the vectors in the output container
@@ -440,12 +440,13 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
}
private void updateStats(HashTable htable) {
+ if(htable == null) return;
htable.getStats(htStats);
this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_BUCKETS, htStats.numBuckets);
this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_ENTRIES, htStats.numEntries);
- this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_RESIZING, htStats.numResizing);
+ this.stats.addLongStat(HashTableMetrics.HTABLE_NUM_RESIZING, htStats.numResizing);
}
-
+
@Override
public void killIncoming() {
this.left.kill();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 84f8354..e32b653 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -166,12 +166,12 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
stats.startSetup();
this.worker = generateNewWorker();
first = true;
- stats.stopSetup();
} catch (ClassTransformationException | IOException | SchemaChangeException e) {
- stats.stopSetup();
context.fail(new SchemaChangeException(e));
kill();
return IterOutcome.STOP;
+ } finally {
+ stats.stopSetup();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/894037ab/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index 72a7d3b..4c1f82d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -66,9 +66,13 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
}
public final IterOutcome next(int inputIndex, RecordBatch b){
+ IterOutcome next = null;
stats.stopProcessing();
- IterOutcome next = b.next();
- stats.startProcessing();
+ try{
+ next = b.next();
+ }finally{
+ stats.startProcessing();
+ }
switch(next){
case OK_NEW_SCHEMA:
@@ -138,7 +142,7 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
return batch;
}
-
+
@Override
public VectorContainer getOutgoingContainer() {
throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
[05/13] git commit: Add alias names date_diff,
date_sub alias to date subtraction functions
Posted by ja...@apache.org.
Add alias names date_diff, date_sub alias to date subtraction functions
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5e357fd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5e357fd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5e357fd5
Branch: refs/heads/master
Commit: 5e357fd5177222b7582688bd0e7a04ed32bb782f
Parents: 1465e11
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Jun 17 01:31:18 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:04:14 2014 -0700
----------------------------------------------------------------------
.../DateDateArithmeticFunctions.java | 2 +-
.../DateIntervalArithmeticFunctions.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5e357fd5/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
index 181acf0..676df67 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
@@ -40,7 +40,7 @@ import org.apache.drill.exec.record.RecordBatch;
import io.netty.buffer.ByteBuf;
@SuppressWarnings("unused")
-@FunctionTemplate(names = {"date_diff", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+@FunctionTemplate(names = {"date_diff", "subtract", "date_sub"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public class G${type}Difference implements DrillSimpleFunc {
@Param ${type}Holder left;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5e357fd5/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
index 634e41a..6fc78eb 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
@@ -132,7 +132,7 @@ public class ${datetype}${intervaltype}Functions {
}
@SuppressWarnings("unused")
- @FunctionTemplate(names = {"date_sub", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"date_sub", "subtract", "date_diff"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
public static class ${datetype}${intervaltype}SubtractFunction implements DrillSimpleFunc {
@Param ${datetype}Holder left;
@Param ${intervaltype}Holder right;
@@ -220,7 +220,7 @@ public class ${datetype}${intervaltype}Functions {
}
@SuppressWarnings("unused")
- @FunctionTemplate(names = {"date_sub", "subtract"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"date_sub", "subtract", "date_diff"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
public static class ${datetype}${intervaltype}SubtractFunction implements DrillSimpleFunc {
@Param ${datetype}Holder left;
@Param ${intervaltype}Holder right;
[08/13] git commit: Improve error messaging for OperatorStats
assertion messages.
Posted by ja...@apache.org.
Improve error messaging for OperatorStats assertion messages.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d7a75dda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d7a75dda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d7a75dda
Branch: refs/heads/master
Commit: d7a75ddaf9e2f2e998b071fc1f3fad1b3579f01c
Parents: e2f57b4
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Jun 17 09:28:09 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:04:39 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/ops/OperatorStats.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d7a75dda/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index 4afea7b..7d1e9dc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -67,40 +67,40 @@ public class OperatorStats {
}
public void startSetup() {
- assert !inSetup;
+ assert !inSetup : "Failure while starting setup. Currently in setup.";
stopProcessing();
inSetup = true;
setupMark = System.nanoTime();
}
public void stopSetup() {
- assert inSetup;
+ assert inSetup : "Failure while stopping setup. Not currently in setup.";
startProcessing();
setupNanos += System.nanoTime() - setupMark;
inSetup = false;
}
public void startProcessing() {
- assert !inProcessing;
+ assert !inProcessing : "Failure while starting processing. Currently in processing.";
processingMark = System.nanoTime();
inProcessing = true;
}
public void stopProcessing() {
- assert inProcessing;
+ assert inProcessing : "Failure while stopping processing. Not currently in processing.";
processingNanos += System.nanoTime() - processingMark;
inProcessing = false;
}
public void startWait() {
- assert !inWait;
+ assert !inWait : "Failure while starting waiting. Currently in waiting.";
stopProcessing();
inWait = true;
waitMark = System.nanoTime();
}
public void stopWait() {
- assert inWait;
+ assert inWait : "Failure while stopping waiting. Currently not in waiting.";
startProcessing();
waitNanos += System.nanoTime() - waitMark;
inWait = false;
[02/13] git commit: DRILL-836: [addendum] Drill needs to return
complex types (e.g., map and array) as a JSON string
Posted by ja...@apache.org.
DRILL-836: [addendum] Drill needs to return complex types (e.g., map and array) as a JSON string
* This contains additional changes to the original patch which was merged.
+ Renamed "flatten" to "complex-to-json"
+ With the new patch, we return VARCHAR instead of VARBINARY.
+ Added test case.
+ Minor code re-factoring.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2e07b0b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2e07b0b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2e07b0b8
Branch: refs/heads/master
Commit: 2e07b0b8ab4fe7fdb1ad51ee47ab10aeec94b7f9
Parents: 4d17aea
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Mon Jun 16 15:01:11 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Tue Jun 17 00:06:41 2014 -0700
----------------------------------------------------------------------
.gitignore | 1 +
.../drill/common/util/DrillStringUtils.java | 176 +++++++++++++++++++
.../org/apache/drill/hbase/BaseHBaseTest.java | 28 ++-
.../drill/hbase/HBaseRecordReaderTest.java | 6 +-
.../org/apache/drill/hbase/HBaseTestsSuite.java | 6 +-
.../drill/hbase/TestHBaseCFAsJSONString.java | 54 ++++++
.../drill/hbase/TestHBaseFilterPushDown.java | 12 +-
.../drill/hbase/TestHBaseProjectPushDown.java | 8 +-
.../resources/bootstrap-storage-plugins.json | 1 +
.../src/main/codegen/includes/vv_imports.ftl | 1 +
.../main/codegen/templates/SqlAccessors.java | 2 +-
.../apache/drill/exec/client/DrillClient.java | 16 +-
.../exec/expr/fn/impl/StringFunctionUtil.java | 71 --------
.../exec/expr/fn/impl/StringFunctions.java | 2 +-
.../exec/physical/config/ComplexToJson.java | 59 +++++++
.../drill/exec/physical/config/Flatten.java | 59 -------
.../impl/project/ComplexToJsonBatchCreator.java | 42 +++++
.../impl/project/FlattenBatchCreator.java | 42 -----
.../impl/project/ProjectBatchCreator.java | 3 +-
.../impl/project/ProjectRecordBatch.java | 62 ++++---
.../planner/fragment/SimpleParallelizer.java | 6 +-
.../planner/physical/ComplexToJsonPrel.java | 69 ++++++++
.../exec/planner/physical/FlattenPrel.java | 61 -------
.../visitor/ComplexToJsonPrelVisitor.java | 40 +++++
.../physical/visitor/FlattenPrelVisitor.java | 40 -----
.../planner/sql/handlers/DefaultSqlHandler.java | 6 +-
.../org/apache/drill/exec/rpc/BasicClient.java | 4 +-
.../org/apache/drill/exec/rpc/RpcException.java | 4 +-
.../apache/drill/exec/rpc/user/UserClient.java | 20 ++-
.../apache/drill/exec/rpc/user/UserServer.java | 8 +-
.../apache/drill/exec/rpc/user/UserSession.java | 65 +++++--
.../org/apache/drill/exec/util/ConvertUtil.java | 7 +-
.../org/apache/drill/exec/util/VectorUtil.java | 21 +--
.../java/org/apache/drill/PlanningBase.java | 5 +-
.../exec/physical/impl/TestOptiqPlans.java | 2 +-
exec/jdbc/pom.xml | 1 +
.../apache/drill/exec/proto/UserBitShared.java | 18 +-
.../exec/proto/beans/CoreOperatorType.java | 4 +-
protocol/src/main/protobuf/UserBitShared.proto | 2 +-
39 files changed, 645 insertions(+), 389 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 1092a7d..838ea6b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
.project
.buildpath
.classpath
+.checkstyle
.settings/
.idea/
*.log
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java b/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
new file mode 100644
index 0000000..96b5776
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import org.apache.commons.lang3.StringEscapeUtils;
+
+import io.netty.buffer.ByteBuf;
+
+public class DrillStringUtils {
+ /**
+ * Unescapes any Java literals found in the {@code String}.
+ * For example, it will turn a sequence of {@code '\'} and
+ * {@code 'n'} into a newline character, unless the {@code '\'}
+ * is preceded by another {@code '\'}.
+ *
+ * @param input the {@code String} to unescape, may be null
+ * @return a new unescaped {@code String}, {@code null} if null string input
+ */
+ public static final String unescapeJava(String input) {
+ return StringEscapeUtils.unescapeJava(input);
+ }
+
+ /**
+ * Escapes the characters in a {@code String} using Java String rules.
+ *
+ * Deals correctly with quotes and control-chars (tab, backslash, cr, ff, etc.)
+ *
+ * So a tab becomes the characters {@code '\\'} and
+ * {@code 't'}.
+ *
+ * Example:
+ * <pre>
+ * input string: He didn't say, "Stop!"
+ * output string: He didn't say, \"Stop!\"
+ * </pre>
+ *
+ * @param input String to escape values in, may be null
+ * @return String with escaped values, {@code null} if null string input
+ */
+ public static final String escapeJava(String input) {
+ return StringEscapeUtils.escapeJava(input);
+ }
+
+ public static final String escapeNewLines(String input) {
+ if (input == null) {
+ return null;
+ }
+ StringBuilder result = new StringBuilder();
+ boolean sawNewline = false;
+ for (int i = 0; i < input.length(); i++) {
+ char curChar = input.charAt(i);
+ if (curChar == '\r' || curChar == '\n') {
+ if (sawNewline) {
+ continue;
+ }
+ sawNewline = true;
+ result.append("\\n");
+ } else {
+ sawNewline = false;
+ result.append(curChar);
+ }
+ }
+ return result.toString();
+ }
+
+ /**
+ * Return a printable representation of a byte buffer, escaping the non-printable
+ * bytes as '\\xNN' where NN is the hexadecimal representation of such bytes.
+ *
+ * This function does not modify the {@code readerIndex} and {@code writerIndex}
+ * of the byte buffer.
+ */
+ public static String toBinaryString(ByteBuf buf, int strStart, int strEnd) {
+ StringBuilder result = new StringBuilder();
+ for (int i = strStart; i < strEnd ; ++i) {
+ appendByte(result, buf.getByte(i));
+ }
+ return result.toString();
+ }
+
+ /**
+ * Return a printable representation of a byte array, escaping the non-printable
+ * bytes as '\\xNN' where NN is the hexadecimal representation of such bytes.
+ */
+ public static String toBinaryString(byte[] buf) {
+ return toBinaryString(buf, 0, buf.length);
+ }
+
+ /**
+ * Return a printable representation of a byte array, escaping the non-printable
+ * bytes as '\\xNN' where NN is the hexadecimal representation of such bytes.
+ */
+ public static String toBinaryString(byte[] buf, int strStart, int strEnd) {
+ StringBuilder result = new StringBuilder();
+ for (int i = strStart; i < strEnd ; ++i) {
+ appendByte(result, buf[i]);
+ }
+ return result.toString();
+ }
+
+ private static void appendByte(StringBuilder result, byte b) {
+ int ch = b & 0xFF;
+ if ( (ch >= '0' && ch <= '9')
+ || (ch >= 'A' && ch <= 'Z')
+ || (ch >= 'a' && ch <= 'z')
+ || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
+ result.append((char)ch);
+ } else {
+ result.append(String.format("\\x%02X", ch));
+ }
+ }
+
+ /**
+ * In-place parsing of a hex encoded binary string.
+ *
+ * This function does not modify the {@code readerIndex} and {@code writerIndex}
+ * of the byte buffer.
+ *
+ * @return Index in the byte buffer just after the last written byte.
+ */
+ public static int parseBinaryString(ByteBuf str, int strStart, int strEnd) {
+ int length = (strEnd - strStart);
+ int dstEnd = strStart;
+ for (int i = strStart; i < length ; i++) {
+ byte b = str.getByte(i);
+ if (b == '\\'
+ && length > i+3
+ && (str.getByte(i+1) == 'x' || str.getByte(i+1) == 'X')) {
+ // ok, take next 2 hex digits.
+ byte hd1 = str.getByte(i+2);
+ byte hd2 = str.getByte(i+3);
+ if (isHexDigit(hd1) && isHexDigit(hd2)) { // [a-fA-F0-9]
+ // turn hex ASCII digit -> number
+ b = (byte) ((toBinaryFromHex(hd1) << 4) + toBinaryFromHex(hd2));
+ i += 3; // skip 3
+ }
+ }
+ str.setByte(dstEnd++, b);
+ }
+ return dstEnd;
+ }
+
+ /**
+ * Takes a ASCII digit in the range A-F0-9 and returns
+ * the corresponding integer/ordinal value.
+ * @param ch The hex digit.
+ * @return The converted hex value as a byte.
+ */
+ private static byte toBinaryFromHex(byte ch) {
+ if ( ch >= 'A' && ch <= 'F' )
+ return (byte) ((byte)10 + (byte) (ch - 'A'));
+ else if ( ch >= 'a' && ch <= 'f' )
+ return (byte) ((byte)10 + (byte) (ch - 'a'));
+ return (byte) (ch - '0');
+ }
+
+ private static boolean isHexDigit(byte c) {
+ return (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9');
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index dbeced3..e6a5474 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -42,6 +42,8 @@ import com.google.common.io.Files;
public class BaseHBaseTest extends BaseTestQuery {
+ private static final String HBASE_STORAGE_PLUGIN_NAME = "hbase";
+
protected static Configuration conf = HBaseConfiguration.create();
protected static HBaseStoragePlugin storagePlugin;
@@ -66,10 +68,11 @@ public class BaseHBaseTest extends BaseTestQuery {
HBaseTestsSuite.configure(true, true);
HBaseTestsSuite.initCluster();
- storagePlugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin("hbase");
+ storagePlugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin(HBASE_STORAGE_PLUGIN_NAME);
storagePluginConfig = storagePlugin.getConfig();
-
+ storagePluginConfig.setEnabled(true);
storagePluginConfig.setZookeeperPort(HBaseTestsSuite.getZookeeperPort());
+ bit.getContext().getStorage().createOrUpdate(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig, true);
}
@AfterClass
@@ -91,20 +94,24 @@ public class BaseHBaseTest extends BaseTestQuery {
.replace("[TABLE_NAME]", tableName);
}
- protected void runPhysicalVerifyCount(String planFile, String tableName, int expectedRowCount) throws Exception{
+ protected void runHBasePhysicalVerifyCount(String planFile, String tableName, int expectedRowCount) throws Exception{
String physicalPlan = getPlanText(planFile, tableName);
List<QueryResultBatch> results = testPhysicalWithResults(physicalPlan);
printResultAndVerifyRowCount(results, expectedRowCount);
}
- protected void runSQLVerifyCount(String sql, int expectedRowCount) throws Exception{
- sql = canonizeSQL(sql);
+ protected List<QueryResultBatch> runHBaseSQLlWithResults(String sql) throws Exception {
+ sql = canonizeHBaseSQL(sql);
System.out.println("Running query:\n" + sql);
- List<QueryResultBatch> results = testSqlWithResults(sql);
+ return testSqlWithResults(sql);
+ }
+
+ protected void runHBaseSQLVerifyCount(String sql, int expectedRowCount) throws Exception{
+ List<QueryResultBatch> results = runHBaseSQLlWithResults(sql);
printResultAndVerifyRowCount(results, expectedRowCount);
}
- private void printResultAndVerifyRowCount(List<QueryResultBatch> results, int expectedRowCount) throws SchemaChangeException {
+ protected int printResult(List<QueryResultBatch> results) throws SchemaChangeException {
int rowCount = 0;
RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
for(QueryResultBatch result : results){
@@ -118,12 +125,17 @@ public class BaseHBaseTest extends BaseTestQuery {
result.release();
}
System.out.println("Total record count: " + rowCount);
+ return rowCount;
+ }
+
+ private void printResultAndVerifyRowCount(List<QueryResultBatch> results, int expectedRowCount) throws SchemaChangeException {
+ int rowCount = printResult(results);
if (expectedRowCount != -1) {
Assert.assertEquals(expectedRowCount, rowCount);
}
}
- protected String canonizeSQL(String sql) {
+ protected String canonizeHBaseSQL(String sql) {
return sql.replace("[TABLE_NAME]", HBaseTestsSuite.TEST_TABLE_1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
index 1462b81..60db266 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
@@ -24,19 +24,19 @@ public class HBaseRecordReaderTest extends BaseHBaseTest {
@Test
public void testLocalDistributed() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical.json";
- runPhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 6);
+ runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 6);
}
@Test
public void testLocalDistributedColumnSelect() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical_column_select.json";
- runPhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 2);
+ runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 2);
}
@Test
public void testLocalDistributedFamilySelect() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical_family_select.json";
- runPhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3);
+ runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
index a24215d..18cf87c 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
@@ -38,7 +38,8 @@ import org.junit.runners.Suite.SuiteClasses;
TestHBaseFilterPushDown.class,
TestHBaseProjectPushDown.class,
TestHBaseRegionScanAssignments.class,
- TestHBaseTableProvider.class
+ TestHBaseTableProvider.class,
+ TestHBaseCFAsJSONString.class
})
public class HBaseTestsSuite {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseTestsSuite.class);
@@ -55,6 +56,9 @@ public class HBaseTestsSuite {
private static volatile AtomicInteger initCount = new AtomicInteger(0);
+ /**
+ * This flag controls whether {@link HBaseTestsSuite} starts a mini HBase cluster to run the unit test.
+ */
private static boolean manageHBaseCluster = System.getProperty("drill.hbase.tests.manageHBaseCluster", "true").equalsIgnoreCase("true");
private static boolean hbaseClusterCreated = false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseCFAsJSONString.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseCFAsJSONString.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseCFAsJSONString.java
new file mode 100644
index 0000000..9cc0356
--- /dev/null
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseCFAsJSONString.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.hbase;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestHBaseCFAsJSONString extends BaseHBaseTest {
+
+ private static DrillClient parent_client;
+
+ @BeforeClass
+ public static void openMyClient() throws Exception {
+ parent_client = client;
+ client = new DrillClient(config, serviceSet.getCoordinator());
+ client.setSupportComplexTypes(false);
+ client.connect();
+ }
+
+ @AfterClass
+ public static void closeClient() throws IOException {
+ if(client != null) client.close();
+ client = parent_client;
+ }
+
+ @Test
+ public void testColumnFamiliesAsJSONString() throws Exception {
+ setColumnWidths(new int[] {112, 12});
+ List<QueryResultBatch> resultList = runHBaseSQLlWithResults("SELECT f, f2 FROM hbase.`[TABLE_NAME]` tableName LIMIT 1");
+ printResult(resultList);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 50dddeb..29e7033 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -24,7 +24,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyEqual() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
@@ -36,7 +36,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyGreaterThan() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
@@ -48,7 +48,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyBetween() throws Exception {
setColumnWidths(new int[] {8, 74, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
@@ -60,7 +60,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownMultiColumns() throws Exception {
setColumnWidths(new int[] {8, 74, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` t\n"
@@ -72,7 +72,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownConvertExpression() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
@@ -84,7 +84,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyLessThanOrEqualTo() throws Exception {
setColumnWidths(new int[] {8, 74, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
index ce6f865..6efe905 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
@@ -24,7 +24,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testRowKeyPushDown() throws Exception{
setColumnWidth(8);
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ "row_key\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
@@ -34,7 +34,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testColumnWith1RowPushDown() throws Exception{
setColumnWidth(6);
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ "t.f2.c7 as `t.f2.c7`\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` t"
@@ -44,7 +44,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testRowKeyAndColumnPushDown() throws Exception{
setColumnWidths(new int[] {8, 9, 6, 2, 6});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ "row_key, t.f.c1*31 as `t.f.c1*31`, t.f.c2 as `t.f.c2`, 5 as `5`, 'abc' as `'abc'`\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` t"
@@ -54,7 +54,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testColumnFamilyPushDown() throws Exception{
setColumnWidths(new int[] {8, 74, 38});
- runSQLVerifyCount("SELECT\n"
+ runHBaseSQLVerifyCount("SELECT\n"
+ "row_key, f, f2\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/contrib/storage-hbase/src/test/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/bootstrap-storage-plugins.json b/contrib/storage-hbase/src/test/resources/bootstrap-storage-plugins.json
index 0e93f7e..3e0e8c0 100644
--- a/contrib/storage-hbase/src/test/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-hbase/src/test/resources/bootstrap-storage-plugins.json
@@ -2,6 +2,7 @@
"storage":{
hbase : {
type:"hbase",
+ enabled: false,
config : {
"hbase.zookeeper.quorum" : "localhost",
"hbase.zookeeper.property.clientPort" : 2181
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 9e16116..872c0b8 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -33,6 +33,7 @@ import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.types.TypeProtos.*;
import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.exec.vector.complex.*;
import org.apache.drill.exec.vector.complex.reader.*;
import org.apache.drill.exec.vector.complex.impl.*;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
index d05b7fd..a5b251e 100644
--- a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
+++ b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
@@ -69,7 +69,7 @@ public class ${name}Accessor extends AbstractSqlAccessor{
<#case "VarBinary">
public String getString(int index) {
byte [] b = ac.get(index);
- return StringFunctionUtil.toBinaryString(io.netty.buffer.Unpooled.wrappedBuffer(b), 0, b.length);
+ return DrillStringUtils.toBinaryString(io.netty.buffer.Unpooled.wrappedBuffer(b), 0, b.length);
}
<#break>
<#case "VarChar">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 7dc7702..7d7f2ce 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -72,6 +72,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
private final BufferAllocator allocator;
private int reconnectTimes;
private int reconnectDelay;
+ private boolean supportComplexTypes = true;
private final boolean ownsZkConnection;
private final boolean ownsAllocator;
@@ -110,7 +111,18 @@ public class DrillClient implements Closeable, ConnectionThrottle{
client.setAutoRead(enableAutoRead);
}
-
+ /**
+ * Sets whether the application is willing to accept complex types (Map, Arrays) in the returned result set.
+ * Default is {@code true}. If set to {@code false}, the complex types are returned as JSON encoded VARCHAR type.
+ *
+ * @throws IllegalStateException if called after a connection has been established.
+ */
+ public void setSupportComplexTypes(boolean supportComplexTypes) {
+ if (connected) {
+ throw new IllegalStateException("Attempted to modify client connection property after connection has been established.");
+ }
+ this.supportComplexTypes = supportComplexTypes;
+ }
/**
* Connects the client to a Drillbit server
@@ -181,7 +193,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
private void connect(DrillbitEndpoint endpoint) throws RpcException {
FutureHandler f = new FutureHandler();
try {
- client.connect(f, endpoint, props);
+ client.setSupportComplexTypes(supportComplexTypes).connect(f, endpoint, props);
f.checkedGet();
} catch (InterruptedException e) {
throw new RpcException(e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
index fbdab8e..844a3e8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
@@ -72,77 +72,6 @@ public class StringFunctionUtil {
return -1;
}
- /**
- * Return a printable representation of a byte buffer, escaping the non-printable
- * bytes as '\\xNN' where NN is the hexadecimal representation of such bytes.
- *
- * This function does not modify the {@code readerIndex} and {@code writerIndex}
- * of the byte buffer.
- */
- public static String toBinaryString(ByteBuf buf, int strStart, int strEnd) {
- StringBuilder result = new StringBuilder();
- for (int i = strStart; i < strEnd ; ++i) {
- int ch = buf.getByte(i) & 0xFF;
- if ( (ch >= '0' && ch <= '9')
- || (ch >= 'A' && ch <= 'Z')
- || (ch >= 'a' && ch <= 'z')
- || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
- result.append((char)ch);
- } else {
- result.append(String.format("\\x%02X", ch));
- }
- }
- return result.toString();
- }
-
- /**
- * In-place parsing of a hex encoded binary string.
- *
- * This function does not modify the {@code readerIndex} and {@code writerIndex}
- * of the byte buffer.
- *
- * @return Index in the byte buffer just after the last written byte.
- */
- public static int parseBinaryString(ByteBuf str, int strStart, int strEnd) {
- int length = (strEnd - strStart);
- int dstEnd = strStart;
- for (int i = strStart; i < length ; i++) {
- byte b = str.getByte(i);
- if (b == '\\'
- && length > i+3
- && (str.getByte(i+1) == 'x' || str.getByte(i+1) == 'X')) {
- // ok, take next 2 hex digits.
- byte hd1 = str.getByte(i+2);
- byte hd2 = str.getByte(i+3);
- if (isHexDigit(hd1) && isHexDigit(hd2)) { // [a-fA-F0-9]
- // turn hex ASCII digit -> number
- b = (byte) ((toBinaryFromHex(hd1) << 4) + toBinaryFromHex(hd2));
- i += 3; // skip 3
- }
- }
- str.setByte(dstEnd++, b);
- }
- return dstEnd;
- }
-
- /**
- * Takes a ASCII digit in the range A-F0-9 and returns
- * the corresponding integer/ordinal value.
- * @param ch The hex digit.
- * @return The converted hex value as a byte.
- */
- private static byte toBinaryFromHex(byte ch) {
- if ( ch >= 'A' && ch <= 'F' )
- return (byte) ((byte)10 + (byte) (ch - 'A'));
- else if ( ch >= 'a' && ch <= 'f' )
- return (byte) ((byte)10 + (byte) (ch - 'a'));
- return (byte) (ch - '0');
- }
-
- private static boolean isHexDigit(byte c) {
- return (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9');
- }
-
private static int utf8CharLen(ByteBuf buffer, int idx) {
byte firstByte = buffer.getByte(idx);
if (firstByte >= 0) { // 1-byte char. First byte is 0xxxxxxx.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
index 51a7dbb..33f2c94 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
@@ -928,7 +928,7 @@ public class StringFunctions{
public void eval() {
out.buffer = in.buffer;
out.start = in.start;
- out.end = org.apache.drill.exec.expr.fn.impl.StringFunctionUtil.parseBinaryString(in.buffer, in.start, in.end);
+ out.end = org.apache.drill.common.util.DrillStringUtils.parseBinaryString(in.buffer, in.start, in.end);
out.buffer.readerIndex(out.start);
out.buffer.writerIndex(out.end);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ComplexToJson.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ComplexToJson.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ComplexToJson.java
new file mode 100644
index 0000000..480f84e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ComplexToJson.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import org.apache.drill.exec.physical.base.AbstractSingle;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("complex-to-json")
+public class ComplexToJson extends AbstractSingle {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexToJson.class);
+
+ @JsonCreator
+ public ComplexToJson(@JsonProperty("child") PhysicalOperator child) {
+ super(child);
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E{
+ return physicalVisitor.visitOp(this, value);
+ }
+
+ @Override
+ protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
+ return new ComplexToJson(child);
+ }
+
+ @Override
+ public SelectionVectorMode getSVMode() {
+ return child.getSVMode();
+ }
+
+ @Override
+ public int getOperatorType() {
+ return CoreOperatorType.COMPLEX_TO_JSON_VALUE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
deleted file mode 100644
index d123d2b..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.config;
-
-import org.apache.drill.exec.physical.base.AbstractSingle;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("flatten")
-public class Flatten extends AbstractSingle {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Flatten.class);
-
- @JsonCreator
- public Flatten(@JsonProperty("child") PhysicalOperator child) {
- super(child);
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E{
- return physicalVisitor.visitOp(this, value);
- }
-
- @Override
- protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
- return new Flatten(child);
- }
-
- @Override
- public SelectionVectorMode getSVMode() {
- return child.getSVMode();
- }
-
- @Override
- public int getOperatorType() {
- return CoreOperatorType.FLATTEN_VALUE;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ComplexToJsonBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ComplexToJsonBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ComplexToJsonBatchCreator.java
new file mode 100644
index 0000000..0df9491
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ComplexToJsonBatchCreator.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.project;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.ComplexToJson;
+import org.apache.drill.exec.physical.config.Project;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+
+import com.google.common.base.Preconditions;
+
+public class ComplexToJsonBatchCreator implements BatchCreator<ComplexToJson> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexToJsonBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, ComplexToJson flatten, List<RecordBatch> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.size() == 1);
+ return new ProjectRecordBatch(new Project(null, flatten.getChild()),
+ children.iterator().next(),
+ context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java
deleted file mode 100644
index 9bea73c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.project;
-
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.config.Flatten;
-import org.apache.drill.exec.physical.config.Project;
-import org.apache.drill.exec.physical.impl.BatchCreator;
-import org.apache.drill.exec.record.RecordBatch;
-
-import com.google.common.base.Preconditions;
-
-public class FlattenBatchCreator implements BatchCreator<Flatten> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenBatchCreator.class);
-
- @Override
- public RecordBatch getBatch(FragmentContext context, Flatten flatten, List<RecordBatch> children) throws ExecutionSetupException {
- Preconditions.checkArgument(children.size() == 1);
- return new ProjectRecordBatch(new Project(null, flatten.getChild()),
- children.iterator().next(),
- context);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
index 929071d..cb1d4f1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
@@ -35,6 +35,5 @@ public class ProjectBatchCreator implements BatchCreator<Project>{
Preconditions.checkArgument(children.size() == 1);
return new ProjectRecordBatch(config, children.iterator().next(), context);
}
-
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index f5a4444..5ee01f1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -25,12 +25,16 @@ import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionCallFactory;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -98,9 +102,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
protected void doWork() {
// VectorUtil.showVectorAccessibleContent(incoming, ",");
int incomingRecordCount = incoming.getRecordCount();
-
+
doAlloc();
-
+
int outputRecords = projector.projectRecords(0, incomingRecordCount, 0);
if (outputRecords < incomingRecordCount) {
setValueCount(outputRecords);
@@ -114,8 +118,8 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
}
this.recordCount = outputRecords;
}
- // In case of complex writer expression, vectors would be added to batch run-time.
- // We have to re-build the schema.
+ // In case of complex writer expression, vectors would be added to batch run-time.
+ // We have to re-build the schema.
if (complexWriters != null) {
container.buildSchema(SelectionVectorMode.NONE);
}
@@ -138,17 +142,17 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
}
this.recordCount = remainingRecordCount;
}
- // In case of complex writer expression, vectors would be added to batch run-time.
- // We have to re-build the schema.
+ // In case of complex writer expression, vectors would be added to batch run-time.
+ // We have to re-build the schema.
if (complexWriters != null) {
container.buildSchema(SelectionVectorMode.NONE);
- }
+ }
}
public void addComplexWriter(ComplexWriter writer) {
complexWriters.add(writer);
}
-
+
private boolean doAlloc() {
//Allocate vv in the allocationVectors.
for(ValueVector v : this.allocationVectors){
@@ -156,17 +160,17 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
if (!v.allocateNewSafe())
return false;
}
-
+
//Allocate vv for complexWriters.
if (complexWriters == null)
return true;
-
+
for (ComplexWriter writer : complexWriters)
writer.allocate();
-
+
return true;
}
-
+
private void setValueCount(int count) {
for(ValueVector v : allocationVectors){
ValueVector.Mutator m = v.getMutator();
@@ -177,9 +181,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
return;
for (ComplexWriter writer : complexWriters)
- writer.setValueCount(count);
+ writer.setValueCount(count);
}
-
+
/** hack to make ref and full work together... need to figure out if this is still necessary. **/
private FieldReference getRef(NamedExpression e){
FieldReference ref = e.getRef();
@@ -259,16 +263,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
container.add(tp.getTo());
transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
// logger.debug("Added transfer.");
- } else if (expr instanceof DrillFuncHolderExpr &&
- ((DrillFuncHolderExpr) expr).isComplexWriterFuncHolder()) {
- // Need to process ComplexWriter function evaluation.
- // Lazy initialization of the list of complex writers, if not done yet.
+ } else if (expr instanceof DrillFuncHolderExpr &&
+ ((DrillFuncHolderExpr) expr).isComplexWriterFuncHolder()) {
+ // Need to process ComplexWriter function evaluation.
+ // Lazy initialization of the list of complex writers, if not done yet.
if (complexWriters == null)
complexWriters = Lists.newArrayList();
-
- // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
+
+ // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
((DrillComplexWriterFuncHolder) ((DrillFuncHolderExpr) expr).getHolder()).setReference(namedExpression.getRef());
- cg.addExpr(expr);
+ cg.addExpr(expr);
} else{
// need to do evaluation.
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
@@ -303,10 +307,18 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
List<NamedExpression> exprs = Lists.newArrayList();
for (MaterializedField field : incoming.getSchema()) {
if (Types.isComplex(field.getType())) {
- exprs.add(new NamedExpression(
- FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", field.getPath(), ExpressionPosition.UNKNOWN),
- new FieldReference(field.getPath()))
- );
+ LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", field.getPath(), ExpressionPosition.UNKNOWN);
+ String castFuncName = CastFunctions.getCastFunc(MinorType.VARCHAR);
+ List<LogicalExpression> castArgs = Lists.newArrayList();
+ castArgs.add(convertToJson); //input_expr
+ /*
+ * We are implicitly casting to VARCHAR so we don't have a max length,
+ * using an arbitrary value. We trim down the size of the stored bytes
+ * to the actual size so this size doesn't really matter.
+ */
+ castArgs.add(new ValueExpressions.LongExpression(65536, null)); //
+ FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN);
+ exprs.add(new NamedExpression(castCall, new FieldReference(field.getPath())));
} else {
exprs.add(new NamedExpression(field.getPath(), new FieldReference(field.getPath())));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index 053580b..0ce480d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -20,9 +20,9 @@ package org.apache.drill.exec.planner.fragment;
import java.util.Collection;
import java.util.List;
-import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
+import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
@@ -158,11 +158,11 @@ public class SimpleParallelizer {
.build();
if (isRootNode) {
- logger.debug("Root fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
+ logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
rootFragment = fragment;
rootOperator = root;
} else {
- logger.debug("Remote fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
+ logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
fragments.add(fragment);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java
new file mode 100644
index 0000000..b6bedb6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.ComplexToJson;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.SingleRel;
+import org.eigenbase.relopt.RelTraitSet;
+
+public class ComplexToJsonPrel extends SingleRel implements Prel {
+
+ public ComplexToJsonPrel(Prel phyRelNode) {
+ super(phyRelNode.getCluster(), phyRelNode.getTraitSet(), phyRelNode);
+ }
+
+ @Override
+ public final RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new ComplexToJsonPrel((Prel) sole(inputs));
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ ComplexToJson p = new ComplexToJson(((Prel) getChild()).getPhysicalOperator(creator));
+ return creator.addMetadata(this, p);
+ }
+
+ @Override
+ public Iterator<Prel> iterator() {
+ return PrelUtil.iter(getChild());
+ }
+
+ @Override
+ public SelectionVectorMode[] getSupportedEncodings() {
+ return SelectionVectorMode.DEFAULT;
+ }
+
+ @Override
+ public SelectionVectorMode getEncoding() {
+ return SelectionVectorMode.NONE;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitPrel(this, value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
deleted file mode 100644
index 3c19bae..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.physical;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.config.Flatten;
-import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
-import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.eigenbase.rel.SingleRel;
-
-public class FlattenPrel extends SingleRel implements Prel {
-
- public FlattenPrel(Prel phyRelNode) {
- super(phyRelNode.getCluster(), phyRelNode.getTraitSet(), phyRelNode);
- }
-
- @Override
- public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
- Flatten p = new Flatten(((Prel) getChild()).getPhysicalOperator(creator));
- return creator.addMetadata(this, p);
- }
-
- @Override
- public Iterator<Prel> iterator() {
- return PrelUtil.iter(getChild());
- }
-
- @Override
- public SelectionVectorMode[] getSupportedEncodings() {
- return SelectionVectorMode.DEFAULT;
- }
-
- @Override
- public SelectionVectorMode getEncoding() {
- return SelectionVectorMode.NONE;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitPrel(this, value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
new file mode 100644
index 0000000..37b2f8b
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical.visitor;
+
+import java.util.Collections;
+
+import org.apache.drill.exec.planner.physical.ComplexToJsonPrel;
+import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.ScreenPrel;
+import org.eigenbase.rel.RelNode;
+
+public class ComplexToJsonPrelVisitor extends BasePrelVisitor<Prel, Void, RuntimeException> {
+
+ private static final ComplexToJsonPrelVisitor INSTANCE = new ComplexToJsonPrelVisitor();
+
+ public static Prel addComplexToJsonPrel(Prel prel) {
+ return prel.accept(INSTANCE, null);
+ }
+
+ @Override
+ public Prel visitScreen(ScreenPrel prel, Void value) throws RuntimeException {
+ return prel.copy(prel.getTraitSet(), Collections.singletonList((RelNode)new ComplexToJsonPrel((Prel)prel.getChild())));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java
deleted file mode 100644
index 5892782..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.physical.visitor;
-
-import java.util.Collections;
-
-import org.apache.drill.exec.planner.physical.FlattenPrel;
-import org.apache.drill.exec.planner.physical.Prel;
-import org.apache.drill.exec.planner.physical.ScreenPrel;
-import org.eigenbase.rel.RelNode;
-
-public class FlattenPrelVisitor extends BasePrelVisitor<Prel, Void, RuntimeException> {
-
- private static final FlattenPrelVisitor INSTANCE = new FlattenPrelVisitor();
-
- public static Prel addFlattenPrel(Prel prel) {
- return prel.accept(INSTANCE, null);
- }
-
- @Override
- public Prel visitScreen(ScreenPrel prel, Void value) throws RuntimeException {
- return prel.copy(prel.getTraitSet(), Collections.singletonList((RelNode)new FlattenPrel((Prel)prel.getChild())));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index 78dadbf..21420df 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -44,7 +44,7 @@ import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
import org.apache.drill.exec.planner.physical.visitor.ExcessiveExchangeIdentifier;
import org.apache.drill.exec.planner.physical.visitor.FinalColumnReorderer;
-import org.apache.drill.exec.planner.physical.visitor.FlattenPrelVisitor;
+import org.apache.drill.exec.planner.physical.visitor.ComplexToJsonPrelVisitor;
import org.apache.drill.exec.planner.physical.visitor.JoinPrelRenameVisitor;
import org.apache.drill.exec.planner.physical.visitor.RelUniqifier;
import org.apache.drill.exec.planner.physical.visitor.SelectionVectorPrelVisitor;
@@ -182,8 +182,8 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
* insert a project which which would convert
*/
if (!context.getSession().isSupportComplexTypes()) {
- logger.debug("Client does not support complex types, add Flatten operator.");
- phyRelNode = FlattenPrelVisitor.addFlattenPrel(phyRelNode);
+ logger.debug("Client does not support complex types, add ComplexToJson operator.");
+ phyRelNode = ComplexToJsonPrelVisitor.addComplexToJsonPrel(phyRelNode);
}
/* 6.)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
index 2a3266a..562fa90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
@@ -90,7 +90,9 @@ public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection
public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator);
public boolean isActive(){
- return connection.getChannel().isActive() ;
+ return connection != null
+ && connection.getChannel() != null
+ && connection.getChannel().isActive() ;
}
protected abstract void validateHandshake(HANDSHAKE_RESPONSE validateHandshake) throws RpcException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
index 9b5eb1d..3d8f02b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
@@ -19,8 +19,8 @@ package org.apache.drill.exec.rpc;
import java.util.concurrent.ExecutionException;
-import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.DrillIOException;
+import org.apache.drill.common.util.DrillStringUtils;
/**
* Parent class for all rpc exceptions.
@@ -38,7 +38,7 @@ public class RpcException extends DrillIOException{
}
private static String format(String message) {
- return StringEscapeUtils.unescapeJava(message);
+ return DrillStringUtils.unescapeJava(message);
}
public RpcException(String message) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
index 277bb0c..ad885f6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
@@ -44,6 +44,8 @@ public class UserClient extends BasicClientWithConnection<RpcType, UserToBitHand
private final QueryResultHandler queryResultHandler = new QueryResultHandler();
+ private boolean supportComplexTypes = true;
+
public UserClient(BufferAllocator alloc, EventLoopGroup eventLoopGroup) {
super(UserRpcConfig.MAPPING, alloc, eventLoopGroup, RpcType.HANDSHAKE, BitToUserHandshake.class, BitToUserHandshake.PARSER);
}
@@ -57,7 +59,7 @@ public class UserClient extends BasicClientWithConnection<RpcType, UserToBitHand
UserToBitHandshake.Builder hsBuilder = UserToBitHandshake.newBuilder()
.setRpcVersion(UserRpcConfig.RPC_VERSION)
.setSupportListening(true)
- .setSupportComplexTypes(true);
+ .setSupportComplexTypes(supportComplexTypes);
if (props != null) {
hsBuilder.setProperties(props);
@@ -104,10 +106,24 @@ public class UserClient extends BasicClientWithConnection<RpcType, UserToBitHand
@Override
protected void finalizeConnection(BitToUserHandshake handshake, BasicClientWithConnection.ServerConnection connection) {
}
-
+
@Override
public ProtobufLengthDecoder getDecoder(BufferAllocator allocator) {
return new UserProtobufLengthDecoder(allocator, OutOfMemoryHandler.DEFAULT_INSTANCE);
}
+ /**
+ * Sets whether the application is willing to accept complex types (Map, Arrays) in the returned result set.
+ * Default is {@code true}. If set to {@code false}, the complex types are returned as JSON encoded VARCHAR type.
+ *
+ * @throws IllegalStateException if called after a connection has been established.
+ */
+ public UserClient setSupportComplexTypes(boolean supportComplexTypes) {
+ if (isActive()) {
+ throw new IllegalStateException("Attempted to modify connection property after connection has been established.");
+ }
+ this.supportComplexTypes = supportComplexTypes;
+ return this;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
index e96ba6c..aaf3c2d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
@@ -111,8 +111,12 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
}
void setUser(UserToBitHandshake inbound) throws IOException {
- session = new UserSession(worker.getSystemOptions(), inbound.getCredentials(), inbound.getProperties());
- session.setSupportComplexTypes(inbound.getSupportComplexTypes());
+ session = UserSession.Builder.newBuilder()
+ .withCredentials(inbound.getCredentials())
+ .withOptionManager(worker.getSystemOptions())
+ .withUserProperties(inbound.getProperties())
+ .setSupportComplexTypes(inbound.getSupportComplexTypes())
+ .build();
}
public UserSession getSession(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
index 18e365e..13414da 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.rpc.user;
-import java.io.IOException;
import java.util.Map;
import net.hydromatic.optiq.SchemaPlus;
@@ -42,18 +41,56 @@ public class UserSession {
private Map<String, String> properties;
private OptionManager options;
- public UserSession(OptionManager systemOptions, UserCredentials credentials, UserProperties properties) throws IOException{
- this.credentials = credentials;
- this.options = new SessionOptionManager(systemOptions);
- this.properties = Maps.newHashMap();
+ public static class Builder {
+ UserSession userSession;
- if (properties == null) return;
- for (int i=0; i<properties.getPropertiesCount(); i++) {
- Property prop = properties.getProperties(i);
- this.properties.put(prop.getKey(), prop.getValue());
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public Builder withCredentials(UserCredentials credentials) {
+ userSession.credentials = credentials;
+ return this;
+ }
+
+ public Builder withOptionManager(OptionManager systemOptions) {
+ userSession.options = new SessionOptionManager(systemOptions);
+ return this;
+ }
+
+ public Builder withUserProperties(UserProperties properties) {
+ userSession.properties = Maps.newHashMap();
+ if (properties != null) {
+ for (int i = 0; i < properties.getPropertiesCount(); i++) {
+ Property prop = properties.getProperties(i);
+ userSession.properties.put(prop.getKey(), prop.getValue());
+ }
+ }
+ return this;
+ }
+
+ public Builder setSupportComplexTypes(boolean supportComplexTypes) {
+ userSession.supportComplexTypes = supportComplexTypes;
+ return this;
+ }
+
+ public UserSession build() {
+ UserSession session = userSession;
+ userSession = null;
+ return session;
+ }
+
+ Builder() {
+ userSession = new UserSession();
}
}
+ private UserSession() { }
+
+ public boolean isSupportComplexTypes() {
+ return supportComplexTypes;
+ }
+
public OptionManager getOptions(){
return options;
}
@@ -62,7 +99,6 @@ public class UserSession {
return user;
}
-
/**
* Update the schema path for the session.
* @param fullPath The desired path to set to.
@@ -107,13 +143,4 @@ public class UserSession {
return schema;
}
- public boolean isSupportComplexTypes() {
- return supportComplexTypes;
- }
-
- public UserSession setSupportComplexTypes(boolean supportComplexType) {
- this.supportComplexTypes = supportComplexType;
- return this;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/util/ConvertUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ConvertUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ConvertUtil.java
index 750885c..ffcb7d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ConvertUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ConvertUtil.java
@@ -23,6 +23,7 @@ import io.netty.buffer.Unpooled;
import java.io.DataInput;
+import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.exec.expr.fn.impl.StringFunctionUtil;
public class ConvertUtil {
@@ -45,7 +46,7 @@ public class ConvertUtil {
int actualLen = (end - start);
if (actualLen != requiredLen) {
throw new IllegalArgumentException(String.format("Wrong length %d(%d-%d) in the buffer '%s', expected %d.",
- actualLen, end, start, StringFunctionUtil.toBinaryString(buffer, start, end), requiredLen));
+ actualLen, end, start, DrillStringUtils.toBinaryString(buffer, start, end), requiredLen));
}
}
@@ -90,7 +91,7 @@ public class ConvertUtil {
int availableBytes = (end-start);
if (availableBytes < getVIntSize(i)) {
throw new NumberFormatException("Expected " + getVIntSize(i) + " bytes but the buffer '"
- + StringFunctionUtil.toBinaryString(buffer, start, end) + "' has only "
+ + DrillStringUtils.toBinaryString(buffer, start, end) + "' has only "
+ availableBytes + " bytes.");
}
buffer.writerIndex(start);
@@ -150,7 +151,7 @@ public class ConvertUtil {
return firstByte;
} else if (availableBytes < len) {
throw new NumberFormatException("Expected " + len + " bytes but the buffer '"
- + StringFunctionUtil.toBinaryString(buffer, start, end) + "' has "
+ + DrillStringUtils.toBinaryString(buffer, start, end) + "' has "
+ availableBytes + " bytes.");
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index 38cd530..68f4550 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -20,6 +20,8 @@ package org.apache.drill.exec.util;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.drill.common.util.DrillStringUtils;
+import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorWrapper;
@@ -84,7 +86,8 @@ public class VectorUtil {
int columnWidth = getColumnWidth(columnWidths, columnIndex);
width += columnWidth + 2;
formats.add("| %-" + columnWidth + "s");
- columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
+ MaterializedField field = vw.getValueVector().getField();
+ columns.add(field.getPath().getAsUnescapedPath() + "<" + field.getType().getMinorType() + ">");
columnIndex++;
}
@@ -107,19 +110,13 @@ public class VectorUtil {
for (VectorWrapper<?> vw : va) {
int columnWidth = getColumnWidth(columnWidths, columnIndex);
Object o = vw.getValueVector().getAccessor().getObject(row);
- if (o == null) {
- //null value
- System.out.printf(formats.get(columnIndex), "");
- }
- else if (o instanceof byte[]) {
- String value = new String((byte[]) o);
- System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1));
- } else if (o instanceof List) {
- System.out.printf("| %s", o);
+ String cellString;
+ if (o instanceof byte[]) {
+ cellString = DrillStringUtils.toBinaryString((byte[]) o);
} else {
- String value = o.toString();
- System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1));
+ cellString = DrillStringUtils.escapeNewLines(String.valueOf(o));
}
+ System.out.printf(formats.get(columnIndex), cellString.length() <= columnWidth ? cellString : cellString.substring(0, columnWidth - 1));
columnIndex++;
}
System.out.printf("|\n");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index a819453..741323b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -100,8 +100,7 @@ public class PlanningBase extends ExecTest{
registry.init();
final FunctionImplementationRegistry functionRegistry = new FunctionImplementationRegistry(config);
final SchemaPlus root = Frameworks.createRootSchema(false);
- registry.getSchemaFactory().registerSchemas(new UserSession(null, null, null).setSupportComplexTypes(true), root);
-
+ registry.getSchemaFactory().registerSchemas(UserSession.Builder.newBuilder().setSupportComplexTypes(true).build(), root);
new NonStrictExpectations() {
@@ -113,7 +112,7 @@ public class PlanningBase extends ExecTest{
context.getFunctionRegistry();
result = functionRegistry;
context.getSession();
- result = new UserSession(null, null, null).setSupportComplexTypes(true);
+ result = UserSession.Builder.newBuilder().setSupportComplexTypes(true).build();
context.getCurrentEndpoint();
result = DrillbitEndpoint.getDefaultInstance();
context.getActiveEndpoints();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index f6200f0..a686fa9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -110,7 +110,7 @@ public class TestOptiqPlans extends ExecTest {
};
RemoteServiceSet lss = RemoteServiceSet.getLocalServiceSet();
DrillbitContext bitContext = new DrillbitContext(DrillbitEndpoint.getDefaultInstance(), context, coord, controller, com, cache, workBus, new LocalPStoreProvider(DrillConfig.create()));
- QueryContext qc = new QueryContext(new UserSession(null, null, null).setSupportComplexTypes(true), QueryId.getDefaultInstance(), bitContext);
+ QueryContext qc = new QueryContext(UserSession.Builder.newBuilder().setSupportComplexTypes(true).build(), QueryId.getDefaultInstance(), bitContext);
PhysicalPlanReader reader = bitContext.getPlanReader();
LogicalPlan plan = reader.readLogicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
PhysicalPlan pp = new BasicOptimizer(DrillConfig.create(), qc, connection).optimize(new BasicOptimizer.BasicOptimizationContext(qc), plan);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/exec/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml
index afaaa6d..f19294f 100644
--- a/exec/jdbc/pom.xml
+++ b/exec/jdbc/pom.xml
@@ -98,6 +98,7 @@
<inherited>true</inherited>
<configuration>
<excludes>
+ <exclude>**/.checkstyle</exclude>
<exclude>**/.buildpath</exclude>
<exclude>**/*.json</exclude>
<exclude>**/git.properties</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index faeba6f..2f4a58c 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -455,9 +455,9 @@ public final class UserBitShared {
*/
INFO_SCHEMA_SUB_SCAN(30, 30),
/**
- * <code>FLATTEN = 31;</code>
+ * <code>COMPLEX_TO_JSON = 31;</code>
*/
- FLATTEN(31, 31),
+ COMPLEX_TO_JSON(31, 31),
;
/**
@@ -585,9 +585,9 @@ public final class UserBitShared {
*/
public static final int INFO_SCHEMA_SUB_SCAN_VALUE = 30;
/**
- * <code>FLATTEN = 31;</code>
+ * <code>COMPLEX_TO_JSON = 31;</code>
*/
- public static final int FLATTEN_VALUE = 31;
+ public static final int COMPLEX_TO_JSON_VALUE = 31;
public final int getNumber() { return value; }
@@ -625,7 +625,7 @@ public final class UserBitShared {
case 28: return TEXT_SUB_SCAN;
case 29: return JSON_SUB_SCAN;
case 30: return INFO_SCHEMA_SUB_SCAN;
- case 31: return FLATTEN;
+ case 31: return COMPLEX_TO_JSON;
default: return null;
}
}
@@ -16530,7 +16530,7 @@ public final class UserBitShared {
"\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmen" +
"tState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATI",
"ON\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCE" +
- "LLED\020\004\022\n\n\006FAILED\020\005*\362\004\n\020CoreOperatorType\022" +
+ "LLED\020\004\022\n\n\006FAILED\020\005*\372\004\n\020CoreOperatorType\022" +
"\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001" +
"\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH" +
"_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITIO" +
@@ -16545,9 +16545,9 @@ public final class UserBitShared {
"_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQ" +
"UET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEX" +
"T_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SU" +
- "B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\013\n\007FL" +
- "ATTEN\020\037B.\n\033org.apache.drill.exec.protoB\r" +
- "UserBitSharedH\001"
+ "B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017CO" +
+ "MPLEX_TO_JSON\020\037B.\n\033org.apache.drill.exec" +
+ ".protoB\rUserBitSharedH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
index abd7b78..0485a95 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
@@ -53,7 +53,7 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
TEXT_SUB_SCAN(28),
JSON_SUB_SCAN(29),
INFO_SCHEMA_SUB_SCAN(30),
- FLATTEN(31);
+ COMPLEX_TO_JSON(31);
public final int number;
@@ -102,7 +102,7 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
case 28: return TEXT_SUB_SCAN;
case 29: return JSON_SUB_SCAN;
case 30: return INFO_SCHEMA_SUB_SCAN;
- case 31: return FLATTEN;
+ case 31: return COMPLEX_TO_JSON;
default: return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2e07b0b8/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index eb56efb..fc6f1b5 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -198,5 +198,5 @@ enum CoreOperatorType {
TEXT_SUB_SCAN = 28;
JSON_SUB_SCAN = 29;
INFO_SCHEMA_SUB_SCAN = 30;
- FLATTEN = 31;
+ COMPLEX_TO_JSON = 31;
}
[11/13] git commit: DRILL-842: Aggregate function for correlation
coefficient calculation
Posted by ja...@apache.org.
DRILL-842: Aggregate function for correlation coefficient calculation
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e170cf0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e170cf0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e170cf0b
Branch: refs/heads/master
Commit: e170cf0b302e8f6b625e0f96a66d70da680c594e
Parents: 7fe8a15
Author: Yash Sharma <ya...@snapdeal.com>
Authored: Tue Jun 17 16:25:01 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:25:01 2014 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/codegen/config.fmpp | 6 +-
.../src/main/codegen/data/CorrelationTypes.tdd | 43 +++++
.../src/main/codegen/data/CovarTypes.tdd | 66 ++++++++
.../templates/CorrelationTypeFunctions.java | 156 +++++++++++++++++++
.../codegen/templates/CovarTypeFunctions.java | 149 ++++++++++++++++++
.../exec/fn/impl/TestAggregateFunction.java | 39 +++--
.../src/test/resources/covariance_input.json | 8 +
.../resources/functions/test_covariance.json | 85 ++++++++++
8 files changed, 534 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp
index 5c0d03b..d00e24a 100644
--- a/exec/java-exec/src/main/codegen/config.fmpp
+++ b/exec/java-exec/src/main/codegen/config.fmpp
@@ -25,6 +25,8 @@ data: {
decimalaggrtypes1: tdd(../data/DecimalAggrTypes1.tdd),
aggrtypes2: tdd(../data/AggrTypes2.tdd),
aggrtypes3: tdd(../data/AggrTypes3.tdd),
+ covarTypes: tdd(../data/CovarTypes.tdd),
+ corrTypes: tdd(../data/CorrelationTypes.tdd),
date: tdd(../data/DateTypes.tdd),
extract: tdd(../data/ExtractTypes.tdd),
parser: tdd(../data/Parser.tdd),
@@ -36,6 +38,4 @@ data: {
}
freemarkerLinks: {
includes: includes/
-}
-
-
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/main/codegen/data/CorrelationTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/CorrelationTypes.tdd b/exec/java-exec/src/main/codegen/data/CorrelationTypes.tdd
new file mode 100644
index 0000000..cc6d2a5
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/data/CorrelationTypes.tdd
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ correlationTypes: [
+ {className: "Correlation", funcName: "corr", aliasName: "correlation", types: [
+ {inputType: "BigInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableBigInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Int", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "SmallInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableSmallInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "TinyInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableTinyInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt1", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt1", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt2", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt2", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableFloat4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Float8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableFloat8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"}
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/main/codegen/data/CovarTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/CovarTypes.tdd b/exec/java-exec/src/main/codegen/data/CovarTypes.tdd
new file mode 100644
index 0000000..d0ec489
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/data/CovarTypes.tdd
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ covarianceTypes: [
+ {className: "CoVarianceSample", funcName: "covar_samp", aliasName: "covariance", types: [
+ {inputType: "BigInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableBigInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Int", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "SmallInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableSmallInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "TinyInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableTinyInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt1", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt1", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt2", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt2", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableFloat4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Float8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableFloat8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"}
+ ]
+ },
+ {className: "CoVariancePopulation", funcName: "covar_pop", aliasName: "", types: [
+ {inputType: "BigInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableBigInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Int", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "SmallInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableSmallInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "TinyInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableTinyInt", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt1", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt1", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt2", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt2", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "UInt8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableUInt8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableFloat4", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "Float8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"},
+ {inputType: "NullableFloat8", outputType: "Float8", movingAverageType: "Float8", movingDeviationType: "Float8", countRunningType: "BigInt"}
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/main/codegen/templates/CorrelationTypeFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CorrelationTypeFunctions.java b/exec/java-exec/src/main/codegen/templates/CorrelationTypeFunctions.java
new file mode 100644
index 0000000..19f9c59
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/CorrelationTypeFunctions.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+
+
+<#list corrTypes.correlationTypes as aggrtype>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gaggr/${aggrtype.className}Functions.java" />
+
+<#include "/@includes/license.ftl" />
+
+<#-- A utility class that is used to generate java code for corr/correlation aggr functions -->
+
+/*
+ * This class is automatically generated from CorrelationTypes.tdd using FreeMarker.
+ */
+
+package org.apache.drill.exec.expr.fn.impl.gaggr;
+
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.SmallIntHolder;
+import org.apache.drill.exec.expr.holders.NullableSmallIntHolder;
+import org.apache.drill.exec.expr.holders.TinyIntHolder;
+import org.apache.drill.exec.expr.holders.NullableTinyIntHolder;
+import org.apache.drill.exec.expr.holders.UInt1Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt1Holder;
+import org.apache.drill.exec.expr.holders.UInt2Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt2Holder;
+import org.apache.drill.exec.expr.holders.UInt4Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt4Holder;
+import org.apache.drill.exec.expr.holders.UInt8Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt8Holder;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+
+@SuppressWarnings("unused")
+
+public class ${aggrtype.className}Functions {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${aggrtype.className}Functions.class);
+
+<#list aggrtype.types as type>
+
+@FunctionTemplate(names = {"${aggrtype.funcName}", "${aggrtype.aliasName}"}, scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+public static class ${type.inputType}${aggrtype.className} implements DrillAggFunc{
+
+ @Param ${type.inputType}Holder xIn;
+ @Param ${type.inputType}Holder yIn;
+
+ @Workspace ${type.movingAverageType}Holder xMean;
+ @Workspace ${type.movingAverageType}Holder yMean;
+ @Workspace ${type.movingAverageType}Holder xyMean;
+
+ @Workspace ${type.movingAverageType}Holder xDev;
+ @Workspace ${type.movingAverageType}Holder yDev;
+
+ @Workspace ${type.movingDeviationType}Holder covar;
+
+ @Workspace ${type.countRunningType}Holder count;
+ @Output ${type.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ xMean = new ${type.movingAverageType}Holder();
+ yMean = new ${type.movingAverageType}Holder();
+ xyMean = new ${type.movingDeviationType}Holder();
+ xDev = new ${type.movingDeviationType}Holder();
+ yDev = new ${type.movingDeviationType}Holder();
+ count = new ${type.countRunningType}Holder();
+ covar = new ${type.movingDeviationType}Holder();
+
+ // Initialize the workspace variables
+ xMean.value = 0;
+ yMean.value = 0;
+ xyMean.value = 0;
+ xDev.value = 0;
+ yDev.value = 0;
+ count.value = 1;
+ covar.value = 0;
+ }
+
+ @Override
+ public void add() {
+ <#if type.inputType?starts_with("Nullable")>
+ sout: {
+ if (xIn.isSet == 0 || yIn.isSet == 0) {
+ // processing nullable input and the value is null, so don't do anything...
+ break sout;
+ }
+ </#if>
+
+ // compute covariance
+ double xOldMean = xMean.value, yOldMean = yMean.value;
+
+ xMean.value += ((xIn.value - xMean.value) / count.value);
+ yMean.value += ((yIn.value - yMean.value) / count.value);
+
+ xDev.value += (xIn.value - xOldMean) * (xIn.value - xMean.value);
+ yDev.value += (yIn.value - yOldMean) * (yIn.value - yMean.value);
+
+ xyMean.value += ((xIn.value * yIn.value) - xyMean.value) / count.value;
+ count.value++;
+ <#if type.inputType?starts_with("Nullable")>
+ } // end of sout block
+ </#if>
+ }
+
+ @Override
+ public void output() {
+ double xVariance = (xDev.value / (count.value - 1));
+ double yVariance = (yDev.value / (count.value - 1));
+ double xyCovariance = (xyMean.value - (xMean.value * yMean.value));
+
+ out.value = xyCovariance / Math.sqrt((xVariance * yVariance));
+ }
+
+ @Override
+ public void reset() {
+ xMean.value = 0;
+ yMean.value = 0;
+ xyMean.value = 0;
+ count.value = 1;
+ covar.value = 0;
+ }
+}
+
+
+</#list>
+}
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/main/codegen/templates/CovarTypeFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CovarTypeFunctions.java b/exec/java-exec/src/main/codegen/templates/CovarTypeFunctions.java
new file mode 100644
index 0000000..b8131c2
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/CovarTypeFunctions.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+
+
+<#list covarTypes.covarianceTypes as aggrtype>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gaggr/${aggrtype.className}Functions.java" />
+
+<#include "/@includes/license.ftl" />
+
+<#-- A utility class that is used to generate java code for covariance functions -->
+
+/*
+ * This class is automatically generated from CovarType.tdd using FreeMarker.
+ */
+
+package org.apache.drill.exec.expr.fn.impl.gaggr;
+
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.SmallIntHolder;
+import org.apache.drill.exec.expr.holders.NullableSmallIntHolder;
+import org.apache.drill.exec.expr.holders.TinyIntHolder;
+import org.apache.drill.exec.expr.holders.NullableTinyIntHolder;
+import org.apache.drill.exec.expr.holders.UInt1Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt1Holder;
+import org.apache.drill.exec.expr.holders.UInt2Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt2Holder;
+import org.apache.drill.exec.expr.holders.UInt4Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt4Holder;
+import org.apache.drill.exec.expr.holders.UInt8Holder;
+import org.apache.drill.exec.expr.holders.NullableUInt8Holder;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+
+@SuppressWarnings("unused")
+
+public class ${aggrtype.className}Functions {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${aggrtype.className}Functions.class);
+
+<#list aggrtype.types as type>
+
+<#if aggrtype.aliasName == "">
+@FunctionTemplate(name = "${aggrtype.funcName}", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+<#else>
+@FunctionTemplate(names = {"${aggrtype.funcName}", "${aggrtype.aliasName}"}, scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+</#if>
+
+public static class ${type.inputType}${aggrtype.className} implements DrillAggFunc{
+
+ @Param ${type.inputType}Holder xIn;
+ @Param ${type.inputType}Holder yIn;
+
+ @Workspace ${type.movingAverageType}Holder xMean;
+ @Workspace ${type.movingAverageType}Holder yMean;
+ @Workspace ${type.movingAverageType}Holder xyMean;
+
+ @Workspace ${type.movingDeviationType}Holder covar;
+
+ @Workspace ${type.countRunningType}Holder count;
+ @Output ${type.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ xMean = new ${type.movingAverageType}Holder();
+ yMean = new ${type.movingAverageType}Holder();
+ xyMean = new ${type.movingDeviationType}Holder();
+ count = new ${type.countRunningType}Holder();
+ covar = new ${type.movingDeviationType}Holder();
+
+ // Initialize the workspace variables
+ xMean.value = 0;
+ yMean.value = 0;
+ xyMean.value = 0;
+ count.value = 1;
+ covar.value = 0;
+ }
+
+ @Override
+ public void add() {
+ <#if type.inputType?starts_with("Nullable")>
+ sout: {
+ if (xIn.isSet == 0 || yIn.isSet == 0) {
+ // processing nullable input and the value is null, so don't do anything...
+ break sout;
+ }
+ </#if>
+
+ // compute covariance
+ xMean.value += ((xIn.value - xMean.value) / count.value);
+ yMean.value += ((yIn.value - yMean.value) / count.value);
+
+ xyMean.value += ((xIn.value * yIn.value) - xyMean.value) / count.value;
+ count.value++;
+ <#if type.inputType?starts_with("Nullable")>
+ } // end of sout block
+ </#if>
+ }
+
+ @Override
+ public void output() {
+ <#if aggrtype.funcName == "covar_pop">
+ out.value = (xyMean.value - (xMean.value * yMean.value));
+ <#elseif aggrtype.funcName == "covar_samp">
+ out.value = (xyMean.value - (xMean.value * yMean.value))*(count.value - 1)/(count.value - 2);
+ </#if>
+ }
+
+ @Override
+ public void reset() {
+ xMean.value = 0;
+ yMean.value = 0;
+ xyMean.value = 0;
+ count.value = 1;
+ covar.value = 0;
+ }
+}
+
+
+</#list>
+}
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
index ffb372d..5e57dc7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
@@ -38,8 +38,9 @@ import com.google.common.base.Charsets;
import com.google.common.io.Files;
public class TestAggregateFunction extends PopUnitTestBase {
- @Test
- public void testSortDate() throws Exception {
+
+ public void runTest(Object[] values, String planPath, String dataPath) throws Throwable {
+
try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
Drillbit bit = new Drillbit(CONFIG, serviceSet);
DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
@@ -48,27 +49,17 @@ public class TestAggregateFunction extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/test_stddev_variance.json"), Charsets.UTF_8)
- .replace("#{TEST_FILE}", "/simple_stddev_variance_input.json"));
+ Files.toString(FileUtils.getResourceAsFile(planPath), Charsets.UTF_8).replace("#{TEST_FILE}", dataPath));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
QueryResultBatch batch = results.get(0);
assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
- Double values[] = {2.0d,
- 2.138089935299395d,
- 2.138089935299395d,
- 4.0d,
- 4.571428571428571d,
- 4.571428571428571d};
-
int i = 0;
for (VectorWrapper<?> v : batchLoader) {
-
- ValueVector.Accessor accessor = v.getValueVector().getAccessor();
- System.out.println(accessor.getObject(0));
- assertEquals((accessor.getObject(0)), values[i++]);
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
+ assertEquals(values[i++], (accessor.getObject(0)));
}
batchLoader.clear();
@@ -77,4 +68,22 @@ public class TestAggregateFunction extends PopUnitTestBase {
}
}
}
+
+ @Test
+ public void testSortDate() throws Throwable {
+ String planPath = "/functions/test_stddev_variance.json";
+ String dataPath = "/simple_stddev_variance_input.json";
+ Double expectedValues[] = {2.0d, 2.138089935299395d, 2.138089935299395d, 4.0d, 4.571428571428571d, 4.571428571428571d};
+
+ runTest(expectedValues, planPath, dataPath);
+ }
+
+ @Test
+ public void testCovarianceCorrelation() throws Throwable {
+ String planPath = "/functions/test_covariance.json";
+ String dataPath = "/covariance_input.json";
+ Double expectedValues[] = {4.571428571428571d, 4.857142857142857d, -6.000000000000002d, 4.0d , 4.25d, -5.250000000000002d, 1.0d, 0.9274260335029677d, -1.0000000000000004d};
+
+ runTest(expectedValues, planPath, dataPath);
}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/test/resources/covariance_input.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/covariance_input.json b/exec/java-exec/src/test/resources/covariance_input.json
new file mode 100644
index 0000000..112aeb4
--- /dev/null
+++ b/exec/java-exec/src/test/resources/covariance_input.json
@@ -0,0 +1,8 @@
+{"A" : 2.0, "B" : 2.0, "C" : 1.0, "D" : 8.0}
+{"A" : 4.0, "B" : 4.0, "C" : 2.0, "D" : 7.0}
+{"A" : 4.0, "B" : 4.0, "C" : 3.0, "D" : 6.0}
+{"A" : 4.0, "B" : 4.0, "C" : 4.0, "D" : 5.0}
+{"A" : 5.0, "B" : 5.0, "C" : 5.0, "D" : 4.0}
+{"A" : 5.0, "B" : 5.0, "C" : 6.0, "D" : 3.0}
+{"A" : 7.0, "B" : 7.0, "C" : 7.0, "D" : 2.0}
+{"A" : 9.0, "B" : 9.0, "C" : 8.0, "D" : 1.0}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e170cf0b/exec/java-exec/src/test/resources/functions/test_covariance.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/test_covariance.json b/exec/java-exec/src/test/resources/functions/test_covariance.json
new file mode 100644
index 0000000..3572090
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/test_covariance.json
@@ -0,0 +1,85 @@
+{
+ "head" : {
+ "version" : 1,
+ "generator" : {
+ "type" : "org.apache.drill.exec.planner.logical.DrillImplementor",
+ "info" : ""
+ },
+ "type" : "APACHE_DRILL_PHYSICAL",
+ "resultMode" : "EXEC"
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"fs-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "classpath:///"},
+ files:["#{TEST_FILE}"]
+ },
+ {
+ "pop" : "project",
+ "@id" : 2,
+ "exprs" : [ {
+ "ref" : "`A`",
+ "expr" : "`A`"
+ },
+ {
+ "ref" : "`B`",
+ "expr" : "`B`"
+ },
+ {
+ "ref" : "`C`",
+ "expr" : "`C`"
+ },
+ {
+ "ref" : "`D`",
+ "expr" : "`D`"
+ } ],
+ "child" : 1
+ }, {
+ "pop" : "streaming-aggregate",
+ "@id" : 3,
+ "child" : 2,
+ "keys" : [ ],
+ "exprs" : [ {
+ "ref" : "`EXPR$1`",
+ "expr" : "covar_samp(`A`, `B`) "
+ },
+ {
+ "ref" : "`EXPR$2`",
+ "expr" : "covar_samp(`A`, `C`) "
+ },
+ {
+ "ref" : "`EXPR$3`",
+ "expr" : "covar_samp(`C`, `D`) "
+ },
+ {
+ "ref" : "`EXPR$4`",
+ "expr" : "covar_pop(`A`, `B`) "
+ },
+ {
+ "ref" : "`EXPR$5`",
+ "expr" : "covar_pop(`A`, `C`) "
+ },
+ {
+ "ref" : "`EXPR$6`",
+ "expr" : "covar_pop(`C`, `D`) "
+ },
+ {
+ "ref" : "`EXPR$7`",
+ "expr" : "corr(`A`, `B`) "
+ },
+ {
+ "ref" : "`EXPR$7`",
+ "expr" : "corr(`A`, `C`) "
+ },
+ {
+ "ref" : "`EXPR$8`",
+ "expr" : "corr(`C`, `D`) "
+ } ]
+ }, {
+ "pop" : "screen",
+ "@id" : 4,
+ "child" : 3
+ } ]
+}
\ No newline at end of file
[10/13] git commit: DRILL-699: Allow case insensitive function names
in queries.
Posted by ja...@apache.org.
DRILL-699: Allow case insensitive function names in queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7fe8a15b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7fe8a15b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7fe8a15b
Branch: refs/heads/master
Commit: 7fe8a15b1e638dcad5787360d5b9b955d4dd2f8c
Parents: 5bd7f5f
Author: vkorukanti <ve...@gmail.com>
Authored: Mon Jun 16 16:52:28 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:05:45 2014 -0700
----------------------------------------------------------------------
.../drill/exec/expr/ExpressionTreeMaterializer.java | 6 +++---
.../expr/fn/DrillFunctionImplementationRegistry.java | 8 +++++++-
.../drill/exec/expr/fn/FunctionGenerationHelper.java | 2 +-
.../expr/fn/HiveFunctionImplementationRegistry.java | 13 +++++--------
.../drill/exec/planner/sql/DrillOperatorTable.java | 2 +-
.../drill/exec/physical/impl/TestSimpleFunctions.java | 2 +-
.../org/apache/drill/jdbc/test/TestFunctionsQuery.java | 11 +++++++++++
7 files changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index ef6e6dc..18609f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -142,7 +142,7 @@ public class ExpressionTreeMaterializer {
FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
DrillFuncHolder matchedFuncHolder =
- resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(call.getName()), call);
+ resolver.getBestMatch(registry.getDrillRegistry().getMethods(call.getName()), call);
if (matchedFuncHolder instanceof DrillComplexWriterFuncHolder && ! allowComplexWriter) {
errorCollector.addGeneralError(call.getPosition(), "Only ProjectRecordBatch could have complex writer function. You are using complex writer function " + call.getName() + " in a non-project operation!");
@@ -192,7 +192,7 @@ public class ExpressionTreeMaterializer {
FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN);
DrillFuncHolder matchedCastFuncHolder = resolver.getBestMatch(
- registry.getDrillRegistry().getMethods().get(castFuncName), castCall);
+ registry.getDrillRegistry().getMethods(castFuncName), castCall);
if (matchedCastFuncHolder == null) {
logFunctionResolutionError(errorCollector, castCall);
@@ -326,7 +326,7 @@ public class ExpressionTreeMaterializer {
FunctionResolver resolver = FunctionResolverFactory.getResolver(funcCall);
DrillFuncHolder matchedConvertToNullableFuncHolder =
- resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(funcName), funcCall);
+ resolver.getBestMatch(registry.getDrillRegistry().getMethods(funcName), funcCall);
if (matchedConvertToNullableFuncHolder == null) {
logFunctionResolutionError(errorCollector, funcCall);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
index fb2f443..8db2abd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.expr.fn;
+import java.util.List;
import java.util.Set;
import org.apache.drill.common.config.DrillConfig;
@@ -39,7 +40,7 @@ public class DrillFunctionImplementationRegistry {
if(holder != null){
// register handle for each name the function can be referred to
String[] names = holder.getRegisteredNames();
- for(String name : names) methods.put(name, holder);
+ for(String name : names) methods.put(name.toLowerCase(), holder);
}else{
logger.warn("Unable to initialize function for class {}", clazz.getName());
}
@@ -56,4 +57,9 @@ public class DrillFunctionImplementationRegistry {
public ArrayListMultimap<String, DrillFuncHolder> getMethods() {
return this.methods;
}
+
+ /** Returns functions with given name. Function name is case insensitive. */
+ public List<DrillFuncHolder> getMethods(String name) {
+ return this.methods.get(name.toLowerCase());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
index 21281be..25eff90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
@@ -58,7 +58,7 @@ public class FunctionGenerationHelper {
argExpressions.add(new HoldingContainerExpression(c));
}
- for (DrillFuncHolder h : registry.getDrillRegistry().getMethods().get(name)) {
+ for (DrillFuncHolder h : registry.getDrillRegistry().getMethods(name)) {
if (h.matches(returnType, argTypes)) {
return new DrillFuncHolderExpr(name, h, argExpressions, ExpressionPosition.UNKNOWN);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
index e5c890e..a72b7de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
@@ -75,12 +75,7 @@ public class HiveFunctionImplementationRegistry {
for(int i=0; i<names.length;i++){
- methods.put(names[i], clazz);
- if (!names[i].toLowerCase().equals(names[i])) {
- // After the Optiq-Drill conversion of function calls, function names are in lowercase
- // and we fail to find them in the map. Add a lowercase name entry.
- methods.put(names[i].toLowerCase(), clazz);
- }
+ methods.put(names[i].toLowerCase(), clazz);
}
}
@@ -107,15 +102,17 @@ public class HiveFunctionImplementationRegistry {
argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMinorType());
}
+ String funcName = call.getName().toLowerCase();
+
// search in GenericUDF list
- for(Class<? extends GenericUDF> clazz: methodsGenericUDF.get(call.getName())) {
+ for(Class<? extends GenericUDF> clazz: methodsGenericUDF.get(funcName)) {
holder = matchAndCreateGenericUDFHolder(clazz, argTypes, argOIs);
if(holder != null)
return holder;
}
// search in UDF list
- for (Class<? extends UDF> clazz : methodsUDF.get(call.getName())) {
+ for (Class<? extends UDF> clazz : methodsUDF.get(funcName)) {
holder = matchAndCreateUDFHolder(call.getName(), clazz, argTypes, argOIs);
if (holder != null)
return holder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
index 772b3b9..29161aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
@@ -84,7 +84,7 @@ public class DrillOperatorTable extends SqlStdOperatorTable {
return;
}
- List<SqlOperator> drillOps = opMap.get(opName.getSimple());
+ List<SqlOperator> drillOps = opMap.get(opName.getSimple().toLowerCase());
if(drillOps != null){
operatorList.addAll(drillOps);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 27a6caa..7adef02 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -143,7 +143,7 @@ public class TestSimpleFunctions extends ExecTest {
ExpressionPosition.UNKNOWN
);
FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
- DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(call.getName()), call);
+ DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getDrillRegistry().getMethods(call.getName()), call);
assertEquals( expectedBestInputMode, matchedFuncHolder.getParmMajorType(0).getMode());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fe8a15b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index d700763..39919a5 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -548,4 +548,15 @@ public class TestFunctionsQuery {
"EXT_INTDAY=45.1\n");
}
+ @Test
+ public void testFunctionCaseInsensitiveNames() throws Exception {
+ String query = "SELECT to_date('2003/07/09', 'yyyy/MM/dd') as col1, " +
+ "TO_DATE('2003/07/09', 'yyyy/MM/dd') as col2, " +
+ "To_DaTe('2003/07/09', 'yyyy/MM/dd') as col3 " +
+ "from cp.`employee.json` LIMIT 1";
+
+ JdbcAssert.withNoDefaultSchema()
+ .sql(query)
+ .returns("col1=2003-07-09; col2=2003-07-09; col3=2003-07-09");
+ }
}
[09/13] git commit: DRILL-1001: Expanded ProfileWrapper content and
formatting.
Posted by ja...@apache.org.
DRILL-1001: Expanded ProfileWrapper content and formatting.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5bd7f5ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5bd7f5ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5bd7f5ff
Branch: refs/heads/master
Commit: 5bd7f5ff2f12958fd51cbf1cb932f3bf71af43da
Parents: d7a75dd
Author: Cliff Buchanan <cb...@maprtech.com>
Authored: Tue Jun 17 09:17:03 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jun 17 16:04:47 2014 -0700
----------------------------------------------------------------------
.../drill/exec/server/rest/ProfileWrapper.java | 402 +++++++++++++------
.../src/main/resources/rest/profile/profile.ftl | 2 +-
2 files changed, 279 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5bd7f5ff/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
index 2952c41..c6cd250 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.server.rest;
-import com.google.common.collect.Lists;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile;
import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile;
@@ -28,16 +29,13 @@ import org.apache.drill.exec.proto.UserBitShared.StreamProfile;
import java.text.DateFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Collections;
-import java.util.Date;
-import java.util.List;
+import java.util.Comparator;
import java.util.Locale;
+import java.util.TreeMap;
public class ProfileWrapper {
-
- NumberFormat format = NumberFormat.getInstance(Locale.US);
- DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
-
public QueryProfile profile;
public ProfileWrapper(QueryProfile profile) {
@@ -51,149 +49,305 @@ public class ProfileWrapper {
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
- builder.append("MAJOR FRAGMENTS\nid\tfirst start\tlast start\tfirst end\tlast end\tmin\tavg\tmax\t(time in ms)\n\n" + listMajorFragments());
- builder.append("\n");
- for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) {
- builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), printOperatorsInMajor(majorProfile)));
+ ArrayList<MajorFragmentProfile> majors = new ArrayList<MajorFragmentProfile>(profile.getFragmentProfileList());
+
+ Collections.sort(majors, Comparators.majorIdCompare);
+ builder.append(queryTimingProfile(majors));
+ for (MajorFragmentProfile m : majors) {
+ builder.append(majorFragmentOperatorProfile(m));
+ }
+ for (MajorFragmentProfile m : majors) {
+ builder.append(majorFragmentTimingProfile(m));
}
- builder.append("\n");
- for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) {
- builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), printMinorFragmentsInMajor(majorProfile)));
+ for (MajorFragmentProfile m : majors) {
+ for (MinorFragmentProfile mi : m.getMinorFragmentProfileList()) {
+ builder.append(minorFragmentOperatorProfile(m.getMajorFragmentId(), mi));
+ }
}
return builder.toString();
}
- public String listMajorFragments() {
- StringBuilder builder = new StringBuilder();
- for (MajorFragmentProfile m : profile.getFragmentProfileList()) {
- List<Long> totalTimes = Lists.newArrayList();
- List<Long> startTimes = Lists.newArrayList();
- List<Long> endTimes = Lists.newArrayList();
- for (MinorFragmentProfile minorFragmentProfile : m.getMinorFragmentProfileList()) {
- totalTimes.add(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime());
- startTimes.add(minorFragmentProfile.getStartTime());
- endTimes.add(minorFragmentProfile.getEndTime());
- }
- long min = Collections.min(totalTimes);
- long max = Collections.max(totalTimes);
- long sum = 0;
- for (Long l : totalTimes) {
- sum += l;
- }
- long firstStart = Collections.min(startTimes);
- long lastStart = Collections.max(startTimes);
- long firstEnd = Collections.min(endTimes);
- long lastEnd = Collections.max(endTimes);
- long avg = sum / totalTimes.size();
- builder.append(String.format("%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", m.getMajorFragmentId(), dateFormat.format(new Date(firstStart)),
- dateFormat.format(new Date(lastStart)), dateFormat.format(new Date(firstEnd)), dateFormat.format(new Date(lastEnd)),
- format.format(min), format.format(avg), format.format(max)));
+ public String queryTimingProfile(ArrayList<MajorFragmentProfile> majors) {
+ final String[] columns = {"id", "minors", "first start", "last start", "first end", "last end", "tmin", "tavg", "tmax"};
+ TableBuilder builder = new TableBuilder("Query Timing Profile", "QueryTimingProfile", columns);
+
+
+ long t0 = 0;
+ for (MajorFragmentProfile m : majors) {
+ ArrayList<MinorFragmentProfile> minors = new ArrayList<MinorFragmentProfile>(m.getMinorFragmentProfileList());
+ final String fmt = " (<a href=\"#MinorFragment" + m.getMajorFragmentId() + "_%1$dOperatorProfile\">%1$d</a>)";
+ int li = minors.size() - 1;
+ double total = 0;
+
+ for (MinorFragmentProfile p : minors) {
+ total += p.getEndTime() - p.getStartTime();
+ }
+
+ builder.appendInteger(m.getMajorFragmentId(), null);
+ builder.appendInteger(minors.size(), null);
+
+ Collections.sort(minors, Comparators.startTimeCompare);
+ if (t0 == 0) {
+ t0 = minors.get(0).getStartTime();
+ }
+ builder.appendMillis(minors.get(0).getStartTime() - t0, String.format(fmt, minors.get(0).getMinorFragmentId()));
+ builder.appendMillis(minors.get(li).getStartTime() - t0,String.format(fmt, minors.get(li).getMinorFragmentId()));
+
+ Collections.sort(minors, Comparators.endTimeCompare);
+ builder.appendMillis(minors.get(0).getEndTime() - t0,String.format(fmt, minors.get(0).getMinorFragmentId()));
+ builder.appendMillis(minors.get(li).getEndTime() - t0, String.format(fmt, minors.get(li).getMinorFragmentId()));
+
+ Collections.sort(minors, Comparators.runTimeCompare);
+ builder.appendMillis(minors.get(0).getEndTime() - minors.get(0).getStartTime(), String.format(fmt, minors.get(0).getMinorFragmentId()));
+ builder.appendMillis((long) (total / minors.size()), null);
+ builder.appendMillis(minors.get(li).getEndTime() - minors.get(li).getStartTime(), String.format(fmt, minors.get(li).getMinorFragmentId()));
}
return builder.toString();
}
- public String printMinorFragmentsInMajor(MajorFragmentProfile majorFragmentProfile) {
- StringBuilder builder = new StringBuilder();
- builder.append("id\tstart\tend\ttotal time (ms)\tmax records\tbatches\n");
- for (MinorFragmentProfile m : majorFragmentProfile.getMinorFragmentProfileList()) {
- long startTime = m.getStartTime();
- long endTime = m.getEndTime();
-
- List<OperatorProfile> operators = m.getOperatorProfileList();
- OperatorProfile biggest = null;
- int biggestIncomingRecords = 0;
- for (OperatorProfile oProfile : operators) {
- if (biggest == null) {
- biggest = oProfile;
- int incomingRecordCount = 0;
- for (StreamProfile streamProfile : oProfile.getInputProfileList()) {
- incomingRecordCount += streamProfile.getRecords();
- }
- biggestIncomingRecords = incomingRecordCount;
- } else {
- int incomingRecordCount = 0;
- for (StreamProfile streamProfile : oProfile.getInputProfileList()) {
- incomingRecordCount += streamProfile.getRecords();
- }
- if (incomingRecordCount > biggestIncomingRecords) {
- biggest = oProfile;
- biggestIncomingRecords = incomingRecordCount;
- }
- }
- }
+ public String majorFragmentTimingProfile(MajorFragmentProfile majorFragmentProfile) {
+ ArrayList<MinorFragmentProfile> minors = new ArrayList<MinorFragmentProfile>(majorFragmentProfile.getMinorFragmentProfileList());
+
+ final String[] columns = {"id", "start", "end", "total time", "max records", "max batches"};
+ TableBuilder builder = new TableBuilder(
+ "Major Fragment #" + majorFragmentProfile.getMajorFragmentId() + " Timing Profile",
+ "MajorFragment" + majorFragmentProfile.getMajorFragmentId() + "TimingProfile",
+ columns);
- int biggestBatches = 0;
- for (StreamProfile sProfile : biggest.getInputProfileList()) {
- biggestBatches += sProfile.getBatches();
+ Collections.sort(minors, Comparators.minorIdCompare);
+ for (MinorFragmentProfile m : minors) {
+ ArrayList<OperatorProfile> ops = new ArrayList<OperatorProfile>(m.getOperatorProfileList());
+ long biggestIncomingRecords = 0;
+ long biggestBatches = 0;
+
+ for (StreamProfile sp : ops.get(0).getInputProfileList()) {
+ biggestIncomingRecords += sp.getRecords();
+ biggestBatches += sp.getBatches();
}
-
- builder.append(String.format("%d\t%s\t%s\t%s\t%s\t%s\n", m.getMinorFragmentId(), dateFormat.format(new Date(startTime)),
- dateFormat.format(new Date(endTime)), format.format(endTime - startTime), biggestIncomingRecords, biggestBatches));
+
+ builder.appendInteger(m.getMinorFragmentId(), null);
+ builder.appendTime(m.getStartTime(), null);
+ builder.appendTime(m.getEndTime(), null);
+ builder.appendMillis(m.getEndTime() - m.getStartTime(), null);
+
+ Collections.sort(ops, Comparators.incomingRecordCompare);
+ builder.appendInteger(biggestIncomingRecords, null);
+ builder.appendInteger(biggestBatches, null);
}
return builder.toString();
}
- public String printOperatorsInMajor(MajorFragmentProfile majorFragmentProfile) {
- StringBuilder builder = new StringBuilder();
- builder.append("id\ttype\tp min\tp avg\tp max\ts min\ts avg\ts max\tw min\tw avg\tw max\n");
- int numOperators = majorFragmentProfile.getMinorFragmentProfile(0).getOperatorProfileCount();
- int numFragments = majorFragmentProfile.getMinorFragmentProfileCount();
- long[][] processing = new long[numOperators + 1][numFragments];
- long[][] setup = new long[numOperators + 1][numFragments];
- long[][] wait = new long[numOperators + 1][numFragments];
- CoreOperatorType[] operatorTypes = new CoreOperatorType[numOperators + 1];
-
- for (int i = 0; i < numFragments; i++) {
- MinorFragmentProfile minorProfile = majorFragmentProfile.getMinorFragmentProfile(i);
- for (int j = 0; j < numOperators; j++) {
- OperatorProfile operatorProfile = minorProfile.getOperatorProfile(j);
- int operatorId = operatorProfile.getOperatorId();
- processing[operatorId][i] = operatorProfile.getProcessNanos();
- setup[operatorId][i] = operatorProfile.getSetupNanos();
- wait[operatorId][i] = operatorProfile.getWaitNanos();
- if (i == 0) {
- operatorTypes[operatorId] = CoreOperatorType.valueOf(operatorProfile.getOperatorType());
+ public String majorFragmentOperatorProfile(MajorFragmentProfile major) {
+ TreeMap<Integer, ArrayList<Pair<OperatorProfile, Integer>>> opmap =
+ new TreeMap<Integer, ArrayList<Pair<OperatorProfile, Integer>>>();
+
+
+
+ final String [] columns = {"id", "type", "setup min", "setup avg", "setup max", "process min", "process avg", "process max", "wait min", "wait avg", "wait max"};
+ TableBuilder builder = new TableBuilder(
+ String.format("Major Fragment #%d Operator Profile", major.getMajorFragmentId()),
+ String.format("MajorFragment%dOperatorProfile", major.getMajorFragmentId()),
+ columns);
+
+
+ for (MinorFragmentProfile m : major.getMinorFragmentProfileList()) {
+ int mid = m.getMinorFragmentId();
+
+ for (OperatorProfile op : m.getOperatorProfileList()) {
+ int opid = op.getOperatorId();
+
+ if (!opmap.containsKey(opid)) {
+ opmap.put(opid, new ArrayList<Pair<OperatorProfile, Integer>>());
}
+ opmap.get(opid).add(new ImmutablePair<OperatorProfile, Integer>(op, mid));
}
}
+
+ for (Integer opid : opmap.keySet()) {
+ ArrayList<Pair<OperatorProfile, Integer>> oplist = opmap.get(opid);
+ final String fmt = " (<a href=\"#MinorFragment" + major.getMajorFragmentId() + "_%1$dOperatorProfile\">%1$d</a>)";
+ int li = oplist.size() - 1;
+ double totalsetup = 0;
+ double totalprocess = 0;
+ double totalwait = 0;
- for (int j = 0; j < numOperators + 1; j++) {
- if (operatorTypes[j] == null) {
- continue;
+ for (Pair<OperatorProfile, Integer> opint : oplist) {
+ totalsetup += opint.getLeft().getSetupNanos();
+ totalprocess += opint.getLeft().getProcessNanos();
+ totalwait += opint.getLeft().getWaitNanos();
}
- long processingMin = Long.MAX_VALUE;
- long processingMax = Long.MIN_VALUE;
- long processingSum = 0;
- long setupMin = Long.MAX_VALUE;
- long setupMax = Long.MIN_VALUE;
- long setupSum = 0;
- long waitMin = Long.MAX_VALUE;
- long waitMax = Long.MIN_VALUE;
- long waitSum = 0;
+
+ builder.appendInteger(oplist.get(0).getLeft().getOperatorId(), null);
+ builder.appendCell(CoreOperatorType.valueOf(oplist.get(0).getLeft().getOperatorType()).toString(), null);
+
+ Collections.sort(oplist, Comparators.setupTimeSort);
+ builder.appendNanos(oplist.get(0).getLeft().getSetupNanos(), String.format(fmt, oplist.get(0).getRight()));
+ builder.appendNanos((long) (totalsetup / oplist.size()), null);
+ builder.appendNanos(oplist.get(li).getLeft().getSetupNanos(), String.format(fmt, oplist.get(li).getRight()));
- for (int i = 0; i < numFragments; i++) {
- processingMin = Math.min(processingMin, processing[j][i]);
- processingMax = Math.max(processingMax, processing[j][i]);
- processingSum += processing[j][i];
+ Collections.sort(opmap.get(opid), Comparators.processTimeSort);
+ builder.appendNanos(oplist.get(0).getLeft().getProcessNanos(), String.format(fmt, oplist.get(0).getRight()));
+ builder.appendNanos((long) (totalprocess / oplist.size()), null);
+ builder.appendNanos(oplist.get(li).getLeft().getProcessNanos(), String.format(fmt, oplist.get(li).getRight()));
+
+ Collections.sort(opmap.get(opid), Comparators.waitTimeSort);
+ builder.appendNanos(oplist.get(0).getLeft().getWaitNanos(), String.format(fmt, oplist.get(0).getRight()));
+ builder.appendNanos((long) (totalwait / oplist.size()), null);
+ builder.appendNanos(oplist.get(li).getLeft().getWaitNanos(), String.format(fmt, oplist.get(li).getRight()));
+ }
+ return builder.toString();
+ }
+
+ public String minorFragmentOperatorProfile(int majorId, MinorFragmentProfile minorFragmentProfile) {
+ ArrayList<OperatorProfile> oplist = new ArrayList<OperatorProfile>(minorFragmentProfile.getOperatorProfileList());
+
+ final String[] columns = {"id", "type", "setup", "process", "wait"};
+ TableBuilder builder = new TableBuilder(
+ String.format("Minor Fragment #%d-%d Operator Profile", majorId, minorFragmentProfile.getMinorFragmentId()),
+ String.format("MinorFragment%d_%dOperatorProfile", majorId, minorFragmentProfile.getMinorFragmentId()),
+ columns);
- setupMin = Math.min(setupMin, setup[j][i]);
- setupMax = Math.max(setupMax, setup[j][i]);
- setupSum += setup[j][i];
+ Collections.sort(oplist, Comparators.operatorIdCompare);
+ for (OperatorProfile op : oplist) {
+ builder.appendInteger(op.getOperatorId(), null);
+ builder.appendCell(CoreOperatorType.valueOf(op.getOperatorType()).toString(), null);
+ builder.appendNanos(op.getSetupNanos(), null);
+ builder.appendNanos(op.getProcessNanos(), null);
+ builder.appendNanos(op.getWaitNanos(), null);
+ }
+
+ return builder.toString();
+ }
- waitMin = Math.min(waitMin, wait[j][i]);
- waitMax = Math.max(waitMax, wait[j][i]);
- waitSum += wait[j][i];
+ private static class Comparators {
+ final static Comparator<MajorFragmentProfile> majorIdCompare = new Comparator<MajorFragmentProfile>() {
+ public int compare(MajorFragmentProfile o1, MajorFragmentProfile o2) {
+ return o1.getMajorFragmentId() < o2.getMajorFragmentId() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<MinorFragmentProfile> minorIdCompare = new Comparator<MinorFragmentProfile>() {
+ public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) {
+ return o1.getMinorFragmentId() < o2.getMinorFragmentId() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<MinorFragmentProfile> startTimeCompare = new Comparator<MinorFragmentProfile>() {
+ public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) {
+ return o1.getStartTime() < o2.getStartTime() ? -1 : 1;
}
+ };
- long processingAvg = processingSum / numFragments;
- long setupAvg = setupSum / numFragments;
- long waitAvg = waitSum / numFragments;
+ final static Comparator<MinorFragmentProfile> endTimeCompare = new Comparator<MinorFragmentProfile>() {
+ public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) {
+ return o1.getEndTime() < o2.getEndTime() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<MinorFragmentProfile> runTimeCompare = new Comparator<MinorFragmentProfile>() {
+ public int compare(MinorFragmentProfile o1, MinorFragmentProfile o2) {
+ return o1.getEndTime() - o1.getStartTime() < o2.getEndTime() - o2.getStartTime() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<OperatorProfile> operatorIdCompare = new Comparator<OperatorProfile>() {
+ public int compare(OperatorProfile o1, OperatorProfile o2) {
+ return o1.getOperatorId() < o2.getOperatorId() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<OperatorProfile> incomingRecordCompare = new Comparator<OperatorProfile>() {
+ public long incomingRecordCount(OperatorProfile op) {
+ long count = 0;
+ for (StreamProfile sp : op.getInputProfileList()) {
+ count += sp.getRecords();
+ }
+ return count;
+ }
+
+ public int compare(OperatorProfile o1, OperatorProfile o2) {
+ return incomingRecordCount(o1) > incomingRecordCount(o2) ? -1 : 1;
+ }
+ };
+
+ final static Comparator<Pair<OperatorProfile, Integer>> setupTimeSort = new Comparator<Pair<OperatorProfile, Integer>>() {
+ public int compare(Pair<OperatorProfile, Integer> o1, Pair<OperatorProfile, Integer> o2) {
+ return o1.getLeft().getSetupNanos() < o2.getLeft().getSetupNanos() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<Pair<OperatorProfile, Integer>> processTimeSort = new Comparator<Pair<OperatorProfile, Integer>>() {
+ public int compare(Pair<OperatorProfile, Integer> o1, Pair<OperatorProfile, Integer> o2) {
+ return o1.getLeft().getProcessNanos() < o2.getLeft().getProcessNanos() ? -1 : 1;
+ }
+ };
+
+ final static Comparator<Pair<OperatorProfile, Integer>> waitTimeSort = new Comparator<Pair<OperatorProfile, Integer>>() {
+ public int compare(Pair<OperatorProfile, Integer> o1, Pair<OperatorProfile, Integer> o2) {
+ return o1.getLeft().getWaitNanos() < o2.getLeft().getWaitNanos() ? -1 : 1;
+ }
+ };
+ }
+
+ class TableBuilder {
+ NumberFormat format = NumberFormat.getInstance(Locale.US);
+ DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
+
+ StringBuilder sb;
+ int w = 0;
+ int width;
+
+ public TableBuilder(String title, String id, String[] columns) {
+ sb = new StringBuilder();
+ width = columns.length;
+
+ format.setMaximumFractionDigits(3);
+ format.setMinimumFractionDigits(3);
+
+ sb.append(String.format("<h3 id=\"%s\">%s</h3>\n", id, title));
+ sb.append("<table class=\"table table-bordered text-right\">\n<tr>");
+ for (String cn : columns) {
+ sb.append("<th>" + cn + "</th>");
+ }
+ sb.append("</tr>\n");
+ }
+
+ public void appendCell(String s, String link) {
+ if (w == 0) {
+ sb.append("<tr>");
+ }
+ sb.append(String.format("<td>%s%s</td>", s, link != null ? link : ""));
+ if (++w >= width) {
+ sb.append("</tr>\n");
+ w = 0;
+ }
+ }
+
+ public void appendTime(long d, String link) {
+ appendCell(dateFormat.format(d), link);
+ }
+
+ public void appendMillis(long p, String link) {
+ appendCell(format.format(p / 1000.0), link);
+ }
+
+ public void appendNanos(long p, String link) {
+ appendMillis((long) (p / 1000.0 / 1000.0), link);
+ }
+
+ public void appendFormattedNumber(Number n, String link) {
+ appendCell(format.format(n), link);
+ }
- builder.append(String.format("%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j, operatorTypes[j].toString(),
- format.format(processingMin/1000/1000), format.format(processingAvg/1000/1000), format.format(processingMax/1000/1000),
- format.format(setupMin/1000/1000), format.format(setupAvg/1000/1000), format.format(setupMax/1000/1000),
- format.format(waitMin/1000/1000), format.format(waitAvg/1000/1000), format.format(waitMax/1000/1000)));
+ public void appendInteger(long l, String link) {
+ appendCell(Long.toString(l), link);
+ }
+
+ public String toString() {
+ String rv;
+ rv = sb.append("\n</table>").toString();
+ sb = null;
+ return rv;
}
- return builder.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5bd7f5ff/exec/java-exec/src/main/resources/rest/profile/profile.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
index c3149ab..94b8aa1 100644
--- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
@@ -51,7 +51,7 @@
<div class="page-header">
<h2>Profile Summary</h2>
</div>
- <p><pre>${model.toString()}</pre></p>
+ <p>${model.toString()}</p>
<div class="page-header">
<h2>Complete Profile</h2>
</div>
[03/13] git commit: Commit dd2ae0ae got lost during merge,
reintroducing the race condition.
Posted by ja...@apache.org.
Commit dd2ae0ae got lost during merge, reintroducing the race condition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5079f8b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5079f8b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5079f8b8
Branch: refs/heads/master
Commit: 5079f8b868841f4f632f5d4ef869defc02bb9c31
Parents: 2e07b0b
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue Jun 17 00:43:02 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Tue Jun 17 00:43:02 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/work/foreman/Foreman.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5079f8b8/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 826a22f..9a67653 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -376,6 +376,9 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
}
+ int totalFragments = 1 + intermediateFragments.size() + leafFragments.size();
+ fragmentManager.getStatus().setTotalFragments(totalFragments);
+ fragmentManager.getStatus().updateCache();
logger.debug("Fragments stored.");
logger.debug("Submitting fragments to run.");
@@ -383,9 +386,6 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
logger.debug("Fragments running.");
state.updateState(QueryState.PENDING, QueryState.RUNNING);
- int totalFragments = 1 + intermediateFragments.size() + leafFragments.size();
- fragmentManager.getStatus().setTotalFragments(totalFragments);
- fragmentManager.getStatus().updateCache();
} catch (Exception e) {
fail("Failure while setting up query.", e);