You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/16 23:31:10 UTC
[01/32] DRILL-968: Use checkstyle plugin to prevent inadvertent use
of shaded Guava classes
Repository: incubator-drill
Updated Branches:
refs/heads/master 4198a17a8 -> f86639c31
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
index 43fce83..ac03f7a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.pop;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
index f4fd7ae..46e721d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
@@ -17,52 +17,25 @@
*/
package org.apache.drill.exec.record.vector;
-import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
-import mockit.Injectable;
-import mockit.NonStrictExpectations;
+import java.util.Iterator;
+import java.util.List;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.TopLevelAllocator;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.PhysicalPlan;
-import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
-import org.apache.drill.exec.physical.impl.ImplCreator;
-import org.apache.drill.exec.physical.impl.SimpleRootExec;
-import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.pop.PopUnitTestBase;
-import org.apache.drill.exec.proto.BitControl;
-import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.vector.BigIntVector;
-import org.apache.drill.exec.vector.Float8Vector;
-
import org.apache.drill.exec.vector.ValueVector;
-import org.junit.Ignore;
import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
-
-import java.util.Iterator;
-import java.util.List;
/* This class tests the existing date types. Simply using date types
* by casting from VarChar, performing basic functions and converting
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/util/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/MiniZooKeeperCluster.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/MiniZooKeeperCluster.java
index 64a15b7..7e9dbbd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/MiniZooKeeperCluster.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/MiniZooKeeperCluster.java
@@ -17,14 +17,12 @@
*/
package org.apache.drill.exec.util;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.zookeeper.server.NIOServerCnxnFactory;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnLog;
-
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.Socket;
@@ -32,6 +30,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.persistence.FileTxnLog;
+
/**
* Mostly Copied from HBase's MiniZooKeeperCluster, but without the Hadoop dependency.
*/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
index 6248f57..b98317f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
@@ -17,17 +17,20 @@
*/
package org.apache.drill.exec.work.batch;
-import com.google.common.base.Stopwatch;
-import org.apache.drill.exec.store.LocalSyncableFileSystem;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.base.Stopwatch;
+
public class FileTest {
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 311b3a5..44cba3d 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -17,38 +17,17 @@
*/
package org.apache.drill.jdbc.test;
-import java.io.File;
-import java.lang.Exception;
-import java.lang.RuntimeException;
-import java.nio.file.Paths;
-import java.sql.*;
+import static org.junit.Assert.assertEquals;
+
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.drill.common.util.TestTools;
-import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
-import org.apache.drill.jdbc.Driver;
-import org.apache.drill.jdbc.JdbcTest;
-import org.junit.Assert;
-import org.junit.BeforeClass;
+
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.TestRule;
import com.google.common.base.Function;
-import com.google.common.base.Stopwatch;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
public class TestJdbcQuery extends JdbcTestQueryBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2fa4778..2844a10 100644
--- a/pom.xml
+++ b/pom.xml
@@ -111,7 +111,7 @@
<executions>
<execution>
<id>rat-checks</id>
- <phase>compile</phase>
+ <phase>validate</phase>
<goals>
<goal>check</goal>
</goals>
@@ -136,6 +136,7 @@
<exclude>**/*.iml</exclude>
<exclude>**/*.tdd</exclude>
<exclude>**/*.project</exclude>
+ <exclude>**/*.checkstyle</exclude>
<exclude>.*/**</exclude>
<exclude>*.patch</exclude>
<exclude>**/*.pb.cc</exclude>
@@ -143,7 +144,30 @@
</excludes>
</configuration>
</plugin>
-
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>2.12.1</version>
+ <executions>
+ <execution>
+ <id>checkstyle-validation</id>
+ <phase>validate</phase>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <failsOnError>true</failsOnError>
+ <consoleOutput>true</consoleOutput>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <configLocation>src/main/resources/checkstyle-config.xml</configLocation>
+ <suppressionsLocation>src/main/resources/checkstyle-suppressions.xml</suppressionsLocation>
+ </configuration>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/src/main/resources/checkstyle-config.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/checkstyle-config.xml b/src/main/resources/checkstyle-config.xml
new file mode 100644
index 0000000..6743466
--- /dev/null
+++ b/src/main/resources/checkstyle-config.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<!DOCTYPE module PUBLIC
+ "-//Puppy Crawl//DTD Check Configuration 1.2//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
+
+<!-- Checkstyle configuration for Apache Drill -->
+
+<module name="Checker">
+
+ <module name="TreeWalker">
+
+ <module name="IllegalImport">
+ <property name="illegalPkgs" value="com.google.hive12,com.beust.jcommander.internal,jersey.repackaged"/>
+ </module>
+
+ <module name="AvoidStarImport">
+ <property name="allowStaticMemberImports" value="true"/>
+ </module>
+
+ </module>
+</module>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/src/main/resources/checkstyle-suppressions.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/checkstyle-suppressions.xml b/src/main/resources/checkstyle-suppressions.xml
new file mode 100644
index 0000000..c30ff09
--- /dev/null
+++ b/src/main/resources/checkstyle-suppressions.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<!DOCTYPE suppressions PUBLIC
+ "-//Puppy Crawl//DTD Suppressions 1.1//EN"
+ "suppressions_1_1.dtd">
+
+<!-- Checkstyle Suppressions for Apache Drill -->
+<suppressions>
+ <suppress files="[\\/]generated-sources[\\/]" checks="AvoidStarImport"/>
+</suppressions>
[29/32] git commit: DRILL-247: Math Functions List of math functions
: rand, cbrt, exp, log(a), log(a, base), pi, degrees,
radians Code formatting Review Comments: Removed rand function,
Moved pi away from codegen Implemented Decimal9 & Decimal18 support f
Posted by ja...@apache.org.
DRILL-247: Math Functions List of math functions : rand,cbrt, exp, log(a), log(a,base), pi, degrees, radians Code formatting Review Comments: Removed rand function, Moved pi away from codegen Implemented Decimal9 & Decimal18 support for math functions Drill:247: Code Cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6b9a1c53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6b9a1c53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6b9a1c53
Branch: refs/heads/master
Commit: 6b9a1c53ea60ac1896fa6224ddacde73bb830dfd
Parents: c022b40
Author: Yash Sharma <ya...@snapdeal.com>
Authored: Sun May 18 12:45:21 2014 +0530
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:36 2014 -0700
----------------------------------------------------------------------
.../src/main/codegen/data/MathFunc.tdd | 109 +++++++++++++++++-
.../main/codegen/templates/MathFunctions.java | 110 +++++++++++++++++++
.../drill/exec/expr/fn/impl/MathFunctions.java | 14 +++
.../exec/fn/impl/TestNewMathFunctions.java | 13 +++
.../functions/testExtendedMathFunctions.json | 44 ++++++++
5 files changed, 285 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b9a1c53/exec/java-exec/src/main/codegen/data/MathFunc.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/MathFunc.tdd b/exec/java-exec/src/main/codegen/data/MathFunc.tdd
index e287122..8180141 100644
--- a/exec/java-exec/src/main/codegen/data/MathFunc.tdd
+++ b/exec/java-exec/src/main/codegen/data/MathFunc.tdd
@@ -14,8 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{
- unaryMathFunctions : [
+{
+
+unaryMathFunctions : [
{className: "Abs", funcName: "abs", javaFunc : "java.lang.Math.abs", types: [
{input: "Int", outputType: "Int", castType: "int"},
{input: "BigInt", outputType: "BigInt", castType: "long"},
@@ -106,7 +107,7 @@
{input: "UInt4", outputType: "Int", castType: "int"},
{input: "UInt8", outputType: "Int", castType: "int"}
]
- }
+ }
],
binaryMathFunctions : [
{className: "Div", funcName: "div", javaFunc : " / ", types: [
@@ -133,11 +134,109 @@
{input: "UInt2", outputType: "UInt2", castType: "char"},
{input: "UInt4", outputType: "UInt4", castType: "int"},
{input: "UInt8", outputType: "UInt8", castType: "long"}
- ]
+ ]
}
-
],
+logBaseMathFunction : [
+ {className: "LogBase", funcName: "log", javaFunc : "java.lang.Math.log", outputType: "Float8", types: [
+ {input: "Int"},
+ {input: "BigInt"},
+ {input: "Float4"},
+ {input: "Float8"},
+ {input: "SmallInt"},
+ {input: "TinyInt"},
+ {input: "UInt1"},
+ {input: "UInt2"},
+ {input: "UInt4"},
+ {input: "UInt8"},
+ {input: "Decimal9"},
+ {input: "Decimal18"}
+ ]
+ }
+],
+extendedUnaryMathFunctions : [
+ {className: "Cbrt", funcName: "cbrt", javaFunc : "java.lang.Math.cbrt", outputType: "Float8", types: [
+ {input: "Int"},
+ {input: "BigInt"},
+ {input: "Float4"},
+ {input: "Float8"},
+ {input: "SmallInt"},
+ {input: "TinyInt"},
+ {input: "UInt1"},
+ {input: "UInt2"},
+ {input: "UInt4"},
+ {input: "UInt8"},
+ {input: "Decimal9"},
+ {input: "Decimal18"}
+ ]
+ },
+
+ {className: "Exp", funcName: "exp", javaFunc : "java.lang.Math.exp", outputType: "Float8", types: [
+ {input: "Int"},
+ {input: "BigInt"},
+ {input: "Float4"},
+ {input: "Float8"},
+ {input: "SmallInt"},
+ {input: "TinyInt"},
+ {input: "UInt1"},
+ {input: "UInt2"},
+ {input: "UInt4"},
+ {input: "UInt8"},
+ {input: "Decimal9"},
+ {input: "Decimal18"}
+ ]
+ },
+
+ {className: "Log", funcName: "log", javaFunc : "java.lang.Math.log", outputType: "Float8", types: [
+ {input: "Int"},
+ {input: "BigInt"},
+ {input: "Float4"},
+ {input: "Float8"},
+ {input: "SmallInt"},
+ {input: "TinyInt"},
+ {input: "UInt1"},
+ {input: "UInt2"},
+ {input: "UInt4"},
+ {input: "UInt8"},
+ {input: "Decimal9"},
+ {input: "Decimal18"}
+ ]
+ },
+
+{className: "Degrees", funcName: "degrees", javaFunc : "java.lang.Math.toDegrees", outputType: "Float8", types: [
+ {input: "Int"},
+ {input: "BigInt"},
+ {input: "Float4"},
+ {input: "Float8"},
+ {input: "SmallInt"},
+ {input: "TinyInt"},
+ {input: "UInt1"},
+ {input: "UInt2"},
+ {input: "UInt4"},
+ {input: "UInt8"},
+ {input: "Decimal9"},
+ {input: "Decimal18"}
+ ]
+ },
+
+{className: "Radians", funcName: "radians", javaFunc : "java.lang.Math.toRadians", outputType: "Float8", types: [
+ {input: "Int"},
+ {input: "BigInt"},
+ {input: "Float4"},
+ {input: "Float8"},
+ {input: "SmallInt"},
+ {input: "TinyInt"},
+ {input: "UInt1"},
+ {input: "UInt2"},
+ {input: "UInt4"},
+ {input: "UInt8"},
+ {input: "Decimal9"},
+ {input: "Decimal18"}
+ ]
+ }
+],
+
trigoMathFunctions : [
{className: "Sin", funcName: "sin", javaFunc : "java.lang.Math.sin", outputType: "Float8", types: [
{input: "Int"},
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b9a1c53/exec/java-exec/src/main/codegen/templates/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MathFunctions.java b/exec/java-exec/src/main/codegen/templates/MathFunctions.java
index ac7cedb..d1ba6e8 100644
--- a/exec/java-exec/src/main/codegen/templates/MathFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/MathFunctions.java
@@ -29,6 +29,8 @@
package org.apache.drill.exec.expr.fn.impl;
+import java.math.BigDecimal;
+
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.common.types.TypeProtos.MinorType;
@@ -98,10 +100,118 @@ public class GMathFunctions{
</#list>
}
+
+
//////////////////////////////////////////////////////////////////////////////////////////////////
//End of GMath Functions
//////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/ExtendedMathFunctions.java" />
+<#include "/@includes/license.ftl" />
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//Functions for Extended Math Functions
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.fn.impl.StringFunctions;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+import java.math.BigDecimal;
+/*
+ * This class is automatically generated from MathFunc.tdd using FreeMarker.
+ */
+
+@SuppressWarnings("unused")
+
+public class ExtendedMathFunctions{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExtendedMathFunctions.class);
+
+ private ExtendedMathFunctions(){}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//Unary Math functions with 1 parameter.
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+<#list mathFunc.extendedUnaryMathFunctions as func>
+
+<#list func.types as type>
+
+@FunctionTemplate(name = "${func.funcName}", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+public static class ${func.className}${type.input} implements DrillSimpleFunc {
+
+ @Param ${type.input}Holder in;
+ @Output ${func.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ }
+
+ public void eval() {
+ <#if type.input?matches("^Decimal[1-9]*")>
+ double dblval = new BigDecimal(in.value).setScale(in.scale).doubleValue();
+ out.value = ${func.javaFunc}(dblval);
+ <#else>
+ out.value = ${func.javaFunc}(in.value);
+ </#if>
+ }
+}
+
+</#list>
+</#list>
+
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//Function to handle Log with base.
+//////////////////////////////////////////////////////////////////////////////////////////////////
+<#list mathFunc.logBaseMathFunction as func>
+<#list func.types as type>
+
+@FunctionTemplate(name = "${func.funcName}", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+public static class ${func.className}${type.input} implements DrillSimpleFunc {
+
+ @Param ${type.input}Holder base;
+ @Param ${type.input}Holder val;
+ @Output ${func.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ }
+
+ public void eval() {
+ <#if type.input?matches("^Decimal[1-9]*")>
+ double dblval = new BigDecimal(val.value).setScale(val.scale).doubleValue();
+ out.value = ${func.javaFunc}(dblval)/ ${func.javaFunc}(base.value);
+ <#else>
+ out.value = ${func.javaFunc}(val.value)/ ${func.javaFunc}(base.value);
+ </#if>
+ }
+}
+</#list>
+</#list>
+
+
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//End of Extended Math Functions
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+
<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/TrigoMathFunctions.java" />
<#include "/@includes/license.ftl" />
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b9a1c53/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index 7d5a3a6..0492a6e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -112,4 +112,18 @@ public class MathFunctions{
out.value = (roundedValue.setScale(decimalDigits, java.math.BigDecimal.ROUND_HALF_UP)).doubleValue();
}
}
+
+ @FunctionTemplate(name = "pi", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ public static class Pi implements DrillSimpleFunc {
+
+ @Output Float8Holder out;
+
+ public void setup(RecordBatch b) {
+ }
+
+ public void eval() {
+ out.value = java.lang.Math.PI;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b9a1c53/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
index 3a67d31..3289601 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
@@ -20,6 +20,9 @@ package org.apache.drill.exec.fn.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+
import mockit.Injectable;
import mockit.NonStrictExpectations;
@@ -116,4 +119,14 @@ public class TestNewMathFunctions {
Object [] expected = new Object[] {Math.sin(45), Math.cos(45), Math.tan(45),Math.asin(45), Math.acos(45), Math.atan(45),Math.sinh(45), Math.cosh(45), Math.tanh(45)};
runTest(bitContext, connection, expected, "functions/testTrigoMathFunctions.json");
}
+
+ @Test
+ public void testExtendedMathFunc(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Throwable{
+ BigDecimal d = new BigDecimal("100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001");
+
+ Object [] expected = new Object[] {Math.cbrt(1000), Math.log(10), (Math.log(64.0)/Math.log(2.0)), Math.exp(10), Math.toDegrees(0.5), Math.toRadians(45.0), Math.PI, Math.cbrt(d.doubleValue()), Math.log(d.doubleValue()), (Math.log(d.doubleValue())/Math.log(2)), Math.exp(d.doubleValue()), Math.toDegrees(d.doubleValue()), Math.toRadians(d.doubleValue())};
+
+ runTest(bitContext, connection, expected, "functions/testExtendedMathFunctions.json");
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b9a1c53/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json b/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json
new file mode 100644
index 0000000..9814cf6
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/testExtendedMathFunctions.json
@@ -0,0 +1,44 @@
+{
+ head : {
+ version : 1,
+ generator : {
+ type : "optiq",
+ info : "na"
+ },
+ type : "APACHE_DRILL_PHYSICAL"
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-sub-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 1, types: [
+ {name: "blue", type: "BIGINT", mode: "REQUIRED"}
+ ]}
+ ]
+ }, {
+ pop : "project",
+ @id : 2,
+ exprs : [
+ { ref : "CBRTVAL", expr : " cbrt(1000) "},
+ { ref : "LOGVAL", expr : " log(10) "},
+ { ref : "LOGVALBASE", expr : " log(2.0, 64.0) "},
+ { ref : "EXP", expr : " exp(10) "},
+ { ref : "DEGREES", expr : " degrees(0.5) "},
+ { ref : "RADIANS", expr : " radians(45.0) "},
+ { ref : "PIVAL", expr : " pi() " },
+ { ref : "CBRTVAL", expr : " cbrt(100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001) "},
+ { ref : "LOGVAL", expr : " log(100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001) "},
+ { ref : "LOGVALBASE", expr : " log(2, 100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001) "},
+ { ref : "EXP", expr : " exp(100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001) "},
+ { ref : "DEGREES", expr : " degrees(100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001) "},
+ { ref : "RADIANS", expr : " radians(100111111111111111111111111111111111.00000000000000000000000000000000000000000000000000001) "}
+ ],
+ child : 1
+ }, {
+ pop : "screen",
+ @id : 3,
+ child : 2
+ } ]
+}
\ No newline at end of file
[30/32] git commit: Switch Infinispan to SYNC mode and add cache
future handling.
Posted by ja...@apache.org.
Switch Infinispan to SYNC mode and add cache future handling.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/1568fc63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/1568fc63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/1568fc63
Branch: refs/heads/master
Commit: 1568fc63e4fa2e6c12cc38012334357fc0439da2
Parents: 6b9a1c5
Author: Steven Phillips <sp...@maprtech.com>
Authored: Mon Jun 9 02:35:09 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 13:50:45 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/cache/DistributedMap.java | 9 +--
.../drill/exec/cache/DistributedMultiMap.java | 4 +-
.../drill/exec/cache/infinispan/ICache.java | 62 ++++++++++++++++----
.../drill/exec/cache/local/LocalCache.java | 53 ++++++++++++++---
.../apache/drill/exec/work/foreman/Foreman.java | 24 ++++++--
5 files changed, 123 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1568fc63/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMap.java
index 2411434..7d3ca9c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMap.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMap.java
@@ -18,15 +18,16 @@
package org.apache.drill.exec.cache;
import java.util.Map;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
public interface DistributedMap<K, V>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DistributedMap.class);
public V get(K key);
- public void put(K key, V value);
- public void delete(K key);
- public void putIfAbsent(K key, V value);
- public void putIfAbsent(K key, V value, long ttl, TimeUnit timeUnit);
+ public Future<V> put(K key, V value);
+ public Future<V> delete(K key);
+ public Future<V> putIfAbsent(K key, V value);
+ public Future<V> putIfAbsent(K key, V value, long ttl, TimeUnit timeUnit);
public Iterable<Map.Entry<K, V>> getLocalEntries();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1568fc63/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMultiMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMultiMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMultiMap.java
index bf06646..214a871 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMultiMap.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMultiMap.java
@@ -18,9 +18,11 @@
package org.apache.drill.exec.cache;
import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Future;
public interface DistributedMultiMap<K, V> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DistributedMultiMap.class);
public Collection<V> get(K key);
- public void put(K key, V value);
+ public Future<Boolean> put(K key, V value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1568fc63/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
index 651fc04..d3b63db 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
@@ -25,6 +25,7 @@ import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Maps;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.Counter;
@@ -54,7 +55,6 @@ import org.jgroups.protocols.COUNTER;
import org.jgroups.protocols.FRAG2;
import org.jgroups.stack.ProtocolStack;
-import com.google.common.collect.Maps;
public class ICache implements DistributedCache{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ICache.class);
@@ -69,7 +69,7 @@ public class ICache implements DistributedCache{
String clusterName = config.getString(ExecConstants.SERVICE_NAME);
this.local = local;
- final CacheMode mode = local ? CacheMode.LOCAL : CacheMode.DIST_ASYNC;
+ final CacheMode mode = local ? CacheMode.LOCAL : CacheMode.DIST_SYNC;
GlobalConfigurationBuilder gcb = new GlobalConfigurationBuilder();
if(!local){
@@ -209,23 +209,23 @@ public class ICache implements DistributedCache{
}
@Override
- public void delete(K key) {
- cache.remove(key);
+ public Future<V> delete(K key) {
+ return cache.removeAsync(key);
}
@Override
- public void put(K key, V value) {
- cache.put(key, value);
+ public Future<V> put(K key, V value) {
+ return cache.putAsync(key, value);
}
@Override
- public void putIfAbsent(K key, V value) {
- cache.putIfAbsent(key, value);
+ public Future<V> putIfAbsent(K key, V value) {
+ return cache.putIfAbsentAsync(key, value);
}
@Override
- public void putIfAbsent(K key, V value, long ttl, TimeUnit timeUnit) {
- cache.putIfAbsent(key, value, ttl, timeUnit);
+ public Future<V> putIfAbsent(K key, V value, long ttl, TimeUnit timeUnit) {
+ return cache.putIfAbsentAsync(key, value, ttl, timeUnit);
}
}
@@ -247,16 +247,52 @@ public class ICache implements DistributedCache{
}
@Override
- public void put(K key, V value) {
- cache.put(key, new DeltaList<V>(value));
+ public Future<Boolean> put(K key, V value) {
+ return new ICacheFuture(cache.putAsync(key, new DeltaList(value)));
+ }
+
+ }
+
+ public static class ICacheFuture implements Future<Boolean> {
+
+ Future future;
+
+ public ICacheFuture(Future future) {
+ this.future = future;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return future.cancel(mayInterruptIfRunning);
}
+ @Override
+ public boolean isCancelled() {
+ return future.isCancelled();
+ }
+
+ @Override
+ public boolean isDone() {
+ return future.isDone();
+ }
+
+ @Override
+ public Boolean get() throws InterruptedException, ExecutionException {
+ future.get();
+ return true;
+ }
+
+ @Override
+ public Boolean get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ future.get(timeout, unit);
+ return true;
+ }
}
- private static class DeltaList<V> extends LinkedList<V> implements DeltaAware, Delta{
+ private static class DeltaList<V> extends LinkedList<V> implements DeltaAware, Delta, List<V> {
/** The serialVersionUID */
private static final long serialVersionUID = 2176345973026460708L;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1568fc63/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
index e61cd76..31ab909 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
@@ -29,8 +29,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.common.config.DrillConfig;
@@ -235,8 +234,43 @@ public class LocalCache implements DistributedCache {
}
@Override
- public void put(K key, V value) {
+ public Future<Boolean> put(K key, V value) {
mmap.put(serialize(key, config.getMode()), serialize(value, config.getMode()));
+ return new LocalCacheFuture(true);
+ }
+ }
+
+ public static class LocalCacheFuture<V> implements Future<V> {
+
+ V value;
+
+ public LocalCacheFuture(V value) {
+ this.value = value;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ return true;
+ }
+
+ @Override
+ public V get() throws InterruptedException, ExecutionException {
+ return value;
+ }
+
+ @Override
+ public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ return value;
}
}
@@ -268,25 +302,30 @@ public class LocalCache implements DistributedCache {
}
@Override
- public void put(K key, V value) {
+ public Future<V> put(K key, V value) {
m.put(serialize(key, config.getMode()), serialize(value, config.getMode()));
+ return new LocalCacheFuture(value);
}
@Override
- public void putIfAbsent(K key, V value) {
+ public Future<V> putIfAbsent(K key, V value) {
m.putIfAbsent(serialize(key, config.getMode()), serialize(value, config.getMode()));
+ return new LocalCacheFuture(value);
}
@Override
- public void delete(K key) {
+ public Future<V> delete(K key) {
+ V value = get(key);
m.remove(serialize(key, config.getMode()));
+ return new LocalCacheFuture(value);
}
@Override
- public void putIfAbsent(K key, V value, long ttl, TimeUnit timeUnit) {
+ public Future<V> putIfAbsent(K key, V value, long ttl, TimeUnit timeUnit) {
m.putIfAbsent(serialize(key, config.getMode()), serialize(value, config.getMode()));
logger.warn("Expiration not implemented in local map cache");
+ return new LocalCacheFuture<V>(value);
}
private class DeserializingTransformer implements Iterator<Map.Entry<K, V>> {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1568fc63/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 3048f77..1629f8c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -21,13 +21,20 @@ import io.netty.buffer.ByteBuf;
import java.io.Closeable;
import java.io.IOException;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.cache.CachedVectorContainer;
import org.apache.drill.exec.cache.DistributedCache.CacheConfig;
import org.apache.drill.exec.cache.DistributedCache.SerializationMode;
import org.apache.drill.exec.coord.DistributedSemaphore;
@@ -347,10 +354,11 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
// store fragments in distributed grid.
logger.debug("Storing fragments");
+ List<Future<PlanFragment>> queue = new LinkedList<>();
for (PlanFragment f : work.getFragments()) {
// store all fragments in grid since they are part of handshake.
- context.getCache().getMap(FRAGMENT_CACHE).put(f.getHandle(), f);
+ queue.add(context.getCache().getMap(FRAGMENT_CACHE).put(f.getHandle(), f));
if (f.getLeafFragment()) {
leafFragments.add(f);
} else {
@@ -358,9 +366,14 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
}
- int totalFragments = 1 + intermediateFragments.size() + leafFragments.size();
- fragmentManager.getStatus().setTotalFragments(totalFragments);
- fragmentManager.getStatus().updateCache();
+ for (Future<PlanFragment> f : queue) {
+ try {
+ f.get(10, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ throw new ExecutionSetupException("failure while storing plan fragments", e);
+ }
+ }
+
logger.debug("Fragments stored.");
logger.debug("Submitting fragments to run.");
@@ -368,6 +381,9 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
logger.debug("Fragments running.");
state.updateState(QueryState.PENDING, QueryState.RUNNING);
+ int totalFragments = 1 + intermediateFragments.size() + leafFragments.size();
+ fragmentManager.getStatus().setTotalFragments(totalFragments);
+ fragmentManager.getStatus().updateCache();
} catch (Exception e) {
fail("Failure while setting up query.", e);
[16/32] git commit: More stats fixes: Ensure we are calling the stats
enabled next() method in the template classes for HashJoin,
HashAggregate and StreamingAggregate.
Posted by ja...@apache.org.
More stats fixes: Ensure we are calling the stats enabled next() method in the template classes for HashJoin, HashAggregate and StreamingAggregate.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/57a86d4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/57a86d4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/57a86d4a
Branch: refs/heads/master
Commit: 57a86d4a2c434d997e610016f2fa266108799c93
Parents: fc1a777
Author: Aman Sinha <as...@maprtech.com>
Authored: Fri Jun 13 18:23:53 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:04:44 2014 -0700
----------------------------------------------------------------------
.../exec/physical/impl/aggregate/HashAggTemplate.java | 6 +++---
.../exec/physical/impl/aggregate/HashAggregator.java | 2 +-
.../physical/impl/aggregate/StreamingAggTemplate.java | 6 +++---
.../physical/impl/aggregate/StreamingAggregator.java | 2 +-
.../drill/exec/physical/impl/join/HashJoinBatch.java | 13 +++++--------
.../drill/exec/physical/impl/join/HashJoinHelper.java | 3 +++
.../drill/exec/physical/impl/join/HashJoinProbe.java | 2 +-
.../exec/physical/impl/join/HashJoinProbeTemplate.java | 8 ++++++--
8 files changed, 23 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index 2fb3f02..f73d46c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -77,7 +77,7 @@ public abstract class HashAggTemplate implements HashAggregator {
private int lastBatchOutputCount = 0;
private RecordBatch incoming;
private BatchSchema schema;
- private RecordBatch outgoing;
+ private HashAggBatch outgoing;
private VectorAllocator[] keyAllocators;
private VectorAllocator[] valueAllocators;
private FragmentContext context;
@@ -166,7 +166,7 @@ public abstract class HashAggTemplate implements HashAggregator {
@Override
- public void setup(HashAggregate hashAggrConfig, FragmentContext context, BufferAllocator allocator, RecordBatch incoming, RecordBatch outgoing,
+ public void setup(HashAggregate hashAggrConfig, FragmentContext context, BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
LogicalExpression[] valueExprs,
List<TypedFieldId> valueFieldIds,
TypedFieldId[] groupByOutFieldIds,
@@ -245,7 +245,7 @@ public abstract class HashAggTemplate implements HashAggregator {
for (VectorWrapper<?> v : incoming) {
v.getValueVector().clear();
}
- IterOutcome out = incoming.next();
+ IterOutcome out = outgoing.next(0, incoming);
if(EXTRA_DEBUG_1) logger.debug("Received IterOutcome of {}", out);
switch(out){
case NOT_YET:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
index 9e6cdb9..641d377 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
@@ -42,7 +42,7 @@ public interface HashAggregator {
}
public abstract void setup(HashAggregate hashAggrConfig, FragmentContext context, BufferAllocator allocator, RecordBatch incoming,
- RecordBatch outgoing, LogicalExpression[] valueExprs,
+ HashAggBatch outgoing, LogicalExpression[] valueExprs,
List<TypedFieldId> valueFieldIds,
TypedFieldId[] keyFieldIds,
VectorAllocator[] keyAllocators, VectorAllocator[] valueAllocators)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
index 65d67dc..e73f21b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
@@ -43,14 +43,14 @@ public abstract class StreamingAggTemplate implements StreamingAggregator {
private int outputCount = 0;
private RecordBatch incoming;
private BatchSchema schema;
- private RecordBatch outgoing;
+ private StreamingAggBatch outgoing;
private VectorAllocator[] allocators;
private FragmentContext context;
private InternalBatch remainderBatch;
@Override
- public void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException {
+ public void setup(FragmentContext context, RecordBatch incoming, StreamingAggBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException {
this.allocators = allocators;
this.context = context;
this.incoming = incoming;
@@ -159,7 +159,7 @@ public abstract class StreamingAggTemplate implements StreamingAggregator {
try{
while(true){
previous = new InternalBatch(incoming);
- IterOutcome out = incoming.next();
+ IterOutcome out = outgoing.next(0, incoming);
if(EXTRA_DEBUG) logger.debug("Received IterOutcome of {}", out);
switch(out){
case NONE:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
index 7c10e9d..52f30ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
@@ -32,7 +32,7 @@ public interface StreamingAggregator {
RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR;
}
- public abstract void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing,
+ public abstract void setup(FragmentContext context, RecordBatch incoming, StreamingAggBatch outgoing,
VectorAllocator[] allocators) throws SchemaChangeException;
public abstract IterOutcome getOutcome();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 684965d..ddc31ee 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -63,9 +63,6 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
public static final long ALLOCATOR_INITIAL_RESERVATION = 1*1024*1024;
public static final long ALLOCATOR_MAX_RESERVATION = 20L*1000*1000*1000;
- private static final int LEFT_INPUT = 0;
- private static final int RIGHT_INPUT = 1;
-
// Probe side record batch
private final RecordBatch left;
@@ -159,7 +156,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
* as well, for the materialization to be successful. This batch will not be used
* till we complete the build phase.
*/
- leftUpstream = next(LEFT_INPUT, left);
+ leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
// Build the hash table, using the build side record batches.
executeBuildPhase();
@@ -207,12 +204,12 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
for (VectorWrapper<?> wrapper : left) {
wrapper.getValueVector().clear();
}
- leftUpstream = next(LEFT_INPUT, left);
+ leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
while (leftUpstream == IterOutcome.OK_NEW_SCHEMA || leftUpstream == IterOutcome.OK) {
for (VectorWrapper<?> wrapper : left) {
wrapper.getValueVector().clear();
}
- leftUpstream = next(LEFT_INPUT, left);
+ leftUpstream = next(HashJoinHelper.LEFT_INPUT, left);
}
}
}
@@ -263,7 +260,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
public void executeBuildPhase() throws SchemaChangeException, ClassTransformationException, IOException {
//Setup the underlying hash table
- IterOutcome rightUpstream = next(RIGHT_INPUT, right);
+ IterOutcome rightUpstream = next(HashJoinHelper.RIGHT_INPUT, right);
boolean moreData = true;
@@ -330,7 +327,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
break;
}
// Get the next record batch
- rightUpstream = next(RIGHT_INPUT, right);
+ rightUpstream = next(HashJoinHelper.RIGHT_INPUT, right);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelper.java
index a634827..aec0f31 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelper.java
@@ -69,6 +69,9 @@ public class HashJoinHelper {
// bits to shift while obtaining batch index from SV4
static final int SHIFT_SIZE = 16;
+ public static final int LEFT_INPUT = 0;
+ public static final int RIGHT_INPUT = 1;
+
public HashJoinHelper(FragmentContext context, BufferAllocator allocator) {
this.context = context;
this.allocator = allocator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java
index 6d20f60..ae70339 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java
@@ -48,7 +48,7 @@ public interface HashJoinProbe {
}
public abstract void setupHashJoinProbe(FragmentContext context, VectorContainer buildBatch, RecordBatch probeBatch,
- int probeRecordCount, RecordBatch outgoing, HashTable hashTable, HashJoinHelper hjHelper,
+ int probeRecordCount, HashJoinBatch outgoing, HashTable hashTable, HashJoinHelper hjHelper,
JoinRelType joinRelType);
public abstract void doSetup(FragmentContext context, VectorContainer buildBatch, RecordBatch probeBatch, RecordBatch outgoing);
public abstract int probeAndProject() throws SchemaChangeException, ClassTransformationException, IOException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/57a86d4a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
index 2dec9ff..a4cc662 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.AbstractRecordBatch;
import org.eigenbase.rel.JoinRelType;
@@ -40,6 +41,8 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
// Join type, INNER, LEFT, RIGHT or OUTER
private JoinRelType joinType;
+
+ private HashJoinBatch outgoingJoinBatch = null;
/* Helper class
* Maintains linked list of build side records with the same key
@@ -74,7 +77,7 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
@Override
public void setupHashJoinProbe(FragmentContext context, VectorContainer buildBatch, RecordBatch probeBatch,
- int probeRecordCount, RecordBatch outgoing, HashTable hashTable,
+ int probeRecordCount, HashJoinBatch outgoing, HashTable hashTable,
HashJoinHelper hjHelper, JoinRelType joinRelType) {
this.probeBatch = probeBatch;
@@ -82,6 +85,7 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
this.recordsToProcess = probeRecordCount;
this.hashTable = hashTable;
this.hjHelper = hjHelper;
+ this.outgoingJoinBatch = outgoing;
doSetup(context, buildBatch, probeBatch, outgoing);
}
@@ -104,7 +108,7 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe {
wrapper.getValueVector().clear();
}
- IterOutcome leftUpstream = probeBatch.next();
+ IterOutcome leftUpstream = outgoingJoinBatch.next(HashJoinHelper.LEFT_INPUT, probeBatch);
switch (leftUpstream) {
case NONE:
[11/32] git commit: DRILL-482 : Return null value if cast function
input is a null expression.
Posted by ja...@apache.org.
DRILL-482 : Return null value if cast function input is a null expression.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/817bea53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/817bea53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/817bea53
Branch: refs/heads/master
Commit: 817bea53995f7d003ffa30d67d2dbf237d430a3c
Parents: bc0bb4d
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Fri Jun 13 13:27:31 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:58:26 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/expr/ExpressionTreeMaterializer.java | 7 +++++--
.../src/test/java/org/apache/drill/TestExampleQueries.java | 8 +++++++-
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/817bea53/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 9b80dc0..ef6e6dc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -459,10 +459,13 @@ public class ExpressionTreeMaterializer {
if(castEqual(e.getPosition(), newMajor, input.getMajorType())) return input; // don't do pointless cast.
- if(newMinor == MinorType.LATE || newMinor == MinorType.NULL){
+ if(newMinor == MinorType.LATE){
// if the type still isn't fully bound, leave as cast expression.
return new CastExpression(input, e.getMajorType(), e.getPosition());
- }else{
+ } else if (newMinor == MinorType.NULL) {
+ // if input is a NULL expression, remove cast expression and return a TypedNullConstant directly.
+ return new TypedNullConstant(Types.optional(e.getMajorType().getMinorType()));
+ } else {
// if the type is fully bound, convert to functioncall and materialze the function.
MajorType type = e.getMajorType();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/817bea53/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 2fcc4fb..783c562 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -23,7 +23,7 @@ import org.junit.Test;
public class TestExampleQueries extends BaseTestQuery{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
-
+
@Test // see DRILL-553
public void testQueryWithNullValues() throws Exception {
test("select count(*) from cp.`customer.json` limit 1");
@@ -188,5 +188,11 @@ public class TestExampleQueries extends BaseTestQuery{
public void testUnionAll6() throws Exception {
test("select n_nationkey, n_regionkey from cp.`tpch/nation.parquet` where n_regionkey = 1 union all select r_regionkey, r_regionkey from cp.`tpch/region.parquet` where r_regionkey = 2");
}
+
+ @Test
+ // cast non-exist column from json file. Should return null value.
+ public void testDrill428() throws Exception {
+ test("select cast(NON_EXIST_COL as varchar(10)) from cp.`employee.json` limit 2; ");
+ }
}
[02/32] DRILL-968: Use checkstyle plugin to prevent inadvertent use
of shaded Guava classes
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
index 379d8e1..86ced35 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
@@ -17,17 +17,20 @@
*/
package org.apache.drill.exec.planner.logical;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
import org.apache.drill.exec.planner.common.DrillUnionRelBase;
import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.UnionRel;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
+import org.eigenbase.rel.UnionRel;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
import org.eigenbase.trace.EigenbaseTrace;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Logger;
-
/**
* Rule that converts a {@link UnionRel} to a {@link DrillUnionRelBase}, implemented by a "union" operation.
*/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
index 8a8cb1f..d783f44 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
@@ -18,7 +18,10 @@
package org.apache.drill.exec.planner.logical;
import org.eigenbase.rel.ValuesRel;
-import org.eigenbase.relopt.*;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
/**
* Rule that converts a {@link ValuesRel} to a Drill "values" operation.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
index 2438d39..0a3fde8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
@@ -48,8 +48,8 @@ import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.type.OperandTypes;
import org.eigenbase.sql.type.ReturnTypes;
-import com.beust.jcommander.internal.Lists;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
public abstract class AggPrelBase extends AggregateRelBase implements Prel{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
index 1020757..e6f1fe1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DescribeTableHandler.java
@@ -18,22 +18,28 @@
package org.apache.drill.exec.planner.sql.handlers;
-import com.google.common.collect.ImmutableList;
+import static org.apache.drill.exec.planner.sql.parser.DrillParserUtil.CHARSET;
+
+import java.util.List;
+
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.parser.DrillParserUtil;
import org.apache.drill.exec.planner.sql.parser.SqlDescribeTable;
import org.apache.drill.exec.store.AbstractSchema;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlLiteral;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlNodeList;
+import org.eigenbase.sql.SqlSelect;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
import org.eigenbase.sql.parser.SqlParserPos;
import org.eigenbase.util.Util;
-import java.util.List;
-
-import static org.apache.drill.exec.planner.sql.parser.DrillParserUtil.CHARSET;
+import com.google.common.collect.ImmutableList;
public class DescribeTableHandler extends DefaultSqlHandler {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowSchemasHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowSchemasHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowSchemasHandler.java
index 2a6dc6a..5e77628 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowSchemasHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowSchemasHandler.java
@@ -18,17 +18,22 @@
package org.apache.drill.exec.planner.sql.handlers;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.parser.DrillParserUtil;
import org.apache.drill.exec.planner.sql.parser.SqlShowSchemas;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlNodeList;
+import org.eigenbase.sql.SqlSelect;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
public class ShowSchemasHandler extends DefaultSqlHandler {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
index 2ee36d9..a1c5aee 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowTablesHandler.java
@@ -18,22 +18,28 @@
package org.apache.drill.exec.planner.sql.handlers;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import static org.apache.drill.exec.planner.sql.parser.DrillParserUtil.CHARSET;
+
+import java.util.List;
+
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.tools.Planner;
import net.hydromatic.optiq.tools.RelConversionException;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.parser.DrillParserUtil;
import org.apache.drill.exec.planner.sql.parser.SqlShowTables;
import org.apache.drill.exec.store.AbstractSchema;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlLiteral;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlNodeList;
+import org.eigenbase.sql.SqlSelect;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
-
-import static org.apache.drill.exec.planner.sql.parser.DrillParserUtil.CHARSET;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
public class ShowTablesHandler extends DefaultSqlHandler {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
index 7be40cb..1c916a7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
@@ -17,16 +17,24 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.planner.sql.handlers.CreateTableHandler;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
-import org.eigenbase.sql.*;
+import org.apache.drill.exec.planner.sql.handlers.CreateTableHandler;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlNodeList;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
public class SqlCreateTable extends DrillSqlCall {
public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_TABLE", SqlKind.OTHER);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
index b362a20..f3c8bde 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
@@ -17,16 +17,25 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ViewHandler;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlNodeList;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
public class SqlCreateView extends DrillSqlCall {
public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_VIEW", SqlKind.OTHER);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
index 30c7e43..be804a9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
@@ -17,15 +17,22 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.planner.sql.handlers.DescribeTableHandler;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
-import org.eigenbase.sql.*;
+import org.apache.drill.exec.planner.sql.handlers.DescribeTableHandler;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* Sql parser tree node to represent statement:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
index bbb3a7f..20a7952 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
@@ -17,15 +17,22 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ViewHandler.DropView;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
public class SqlDropView extends DrillSqlCall {
public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DROP_VIEW", SqlKind.OTHER);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
index f882ba9..7e967b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
@@ -17,16 +17,22 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ShowFileHandler;
-import org.apache.drill.exec.planner.sql.handlers.ShowTablesHandler;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* Sql parse tree node to represent statement:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
index 34695d9..2ef6553 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
@@ -17,15 +17,21 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.planner.sql.handlers.ShowSchemasHandler;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
-import org.eigenbase.sql.*;
+import org.apache.drill.exec.planner.sql.handlers.ShowSchemasHandler;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* Sql parse tree node to represent statement:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
index 26d4fa2..93de127 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
@@ -17,15 +17,22 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.planner.sql.handlers.ShowTablesHandler;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
-import org.eigenbase.sql.*;
+import org.apache.drill.exec.planner.sql.handlers.ShowTablesHandler;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* Sql parse tree node to represent statement:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
index 42a3914..554bfbf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
@@ -17,15 +17,22 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
import net.hydromatic.optiq.tools.Planner;
+
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.UseSchemaHandler;
-import org.eigenbase.sql.*;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.SqlSpecialOperator;
+import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
/**
* Sql parser tree node to represent <code>USE SCHEMA</code> statement.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
index 178ac43..84595c6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
@@ -26,7 +26,13 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.BitControl.BitControlHandshake;
import org.apache.drill.exec.proto.BitControl.RpcType;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.rpc.*;
+import org.apache.drill.exec.rpc.BasicClient;
+import org.apache.drill.exec.rpc.OutOfMemoryHandler;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.RpcBus;
+import org.apache.drill.exec.rpc.RpcConnectionHandler;
+import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.work.batch.ControlMessageHandler;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
index 3e1a2a4..fc1d98c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
@@ -25,7 +25,11 @@ import io.netty.util.concurrent.GenericFutureListener;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.BitControl.BitControlHandshake;
import org.apache.drill.exec.proto.BitControl.RpcType;
-import org.apache.drill.exec.rpc.*;
+import org.apache.drill.exec.rpc.BasicServer;
+import org.apache.drill.exec.rpc.OutOfMemoryHandler;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.work.batch.ControlMessageHandler;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
index e22df7c..ed51264 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
@@ -27,9 +27,11 @@ import org.apache.drill.exec.proto.BitData.BitClientHandshake;
import org.apache.drill.exec.proto.BitData.BitServerHandshake;
import org.apache.drill.exec.proto.BitData.RpcType;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.proto.UserBitShared.RpcChannel;
-import org.apache.drill.exec.rpc.*;
-import org.apache.drill.exec.rpc.control.ControlProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.BasicClient;
+import org.apache.drill.exec.rpc.OutOfMemoryHandler;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.server.BootStrapContext;
import com.google.protobuf.MessageLite;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
index 7354d72..ad57956 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
@@ -30,7 +30,11 @@ import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
import org.apache.drill.exec.proto.BitData.RpcType;
import org.apache.drill.exec.proto.UserBitShared.RpcChannel;
import org.apache.drill.exec.record.RawFragmentBatch;
-import org.apache.drill.exec.rpc.*;
+import org.apache.drill.exec.rpc.BasicServer;
+import org.apache.drill.exec.rpc.OutOfMemoryHandler;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.server.BootStrapContext;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
index 31bfd4e..0e3cc6b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
@@ -30,7 +30,12 @@ import org.apache.drill.exec.proto.UserProtos.RpcType;
import org.apache.drill.exec.proto.UserProtos.RunQuery;
import org.apache.drill.exec.proto.UserProtos.UserProperties;
import org.apache.drill.exec.proto.UserProtos.UserToBitHandshake;
-import org.apache.drill.exec.rpc.*;
+import org.apache.drill.exec.rpc.BasicClientWithConnection;
+import org.apache.drill.exec.rpc.OutOfMemoryHandler;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.RpcConnectionHandler;
+import org.apache.drill.exec.rpc.RpcException;
import com.google.protobuf.MessageLite;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
index 3ee25d9..2a4c2cc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
@@ -34,7 +34,14 @@ import org.apache.drill.exec.proto.UserProtos.RpcType;
import org.apache.drill.exec.proto.UserProtos.RunQuery;
import org.apache.drill.exec.proto.UserProtos.UserProperties;
import org.apache.drill.exec.proto.UserProtos.UserToBitHandshake;
-import org.apache.drill.exec.rpc.*;
+import org.apache.drill.exec.rpc.Acks;
+import org.apache.drill.exec.rpc.BasicServer;
+import org.apache.drill.exec.rpc.OutOfMemoryHandler;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
+import org.apache.drill.exec.rpc.RemoteConnection;
+import org.apache.drill.exec.rpc.Response;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.RpcOutcomeListener;
import org.apache.drill.exec.work.user.UserWorker;
import com.google.protobuf.InvalidProtocolBufferException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
index 8d525f3..1e55c73 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
@@ -17,15 +17,30 @@
*/
package org.apache.drill.exec.store;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Progressable;
-import java.io.*;
-import java.net.URI;
-import java.net.URISyntaxException;
-
/**
* This class provides a Syncable local extension of the hadoop FileSystem
*/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
index 1d6aa4d..ef26d11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
@@ -17,7 +17,11 @@
*/
package org.apache.drill.exec.store;
-import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
+import org.apache.drill.exec.vector.RepeatedMutator;
+import org.apache.drill.exec.vector.RepeatedVariableWidthVector;
+import org.apache.drill.exec.vector.ValueVector;
public class VectorHolder {
private int count;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index c03e097..2f0e854 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -38,11 +38,10 @@ import org.apache.drill.exec.store.LocalSyncableFileSystem;
import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
import org.apache.drill.exec.store.dfs.shim.FileSystemCreator;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import com.google.common.collect.Lists;
-import com.beust.jcommander.internal.Maps;
-
-import org.apache.hadoop.fs.FileSystem;
+import com.google.common.collect.Maps;
/**
* A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
index 8809706..b50ada8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
@@ -17,36 +17,36 @@
*/
package org.apache.drill.exec.store.hive;
-import com.google.common.collect.Lists;
-import com.google.common.io.ByteArrayDataInput;
-import com.google.common.io.ByteArrayDataOutput;
-import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.commons.codec.binary.Base64;
-import org.apache.drill.common.util.DataOutputOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.hbase.HBaseSerDe;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.serializer.WritableSerialization;
-import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.Constructor;
-import java.util.*;
+import com.google.common.collect.Lists;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
public class HiveInputReader {
public static void main(String args[]) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
index ecd952c..191af6b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
@@ -17,20 +17,19 @@
*/
package org.apache.drill.exec.store.hive;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.io.ByteArrayDataInput;
-import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
import org.apache.commons.codec.binary.Base64;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.physical.base.AbstractBase;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -38,9 +37,14 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteStreams;
@JsonTypeName("hive-sub-scan")
public class HiveSubScan extends AbstractBase implements SubScan {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
index df2e428..50c81e9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
@@ -17,15 +17,21 @@
*/
package org.apache.drill.exec.store.hive;
-import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.hadoop.hive.metastore.api.*;
-
-import java.util.List;
-import java.util.Map;
+import com.google.common.collect.Lists;
@JsonTypeName("table")
public class HiveTable {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
index 125ee13..b72acf5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
@@ -17,23 +17,25 @@
*/
package org.apache.drill.exec.store.hive;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.InputSplit;
-import java.io.IOException;
-import java.util.List;
+import com.google.common.collect.Lists;
public class HiveTextRecordReader extends HiveRecordReader {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
index a0caf1e..e6d39ed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.store.ischema;
-import com.beust.jcommander.internal.Lists;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -26,7 +26,8 @@ import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeFactory;
import org.eigenbase.sql.type.SqlTypeName;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
/** Base class of tables in INFORMATION_SCHEMA. Defines the table (fields and types) */
public abstract class InfoSchemaTable{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReaders.java
index 6dd96de..76cc937 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReaders.java
@@ -18,15 +18,14 @@
package org.apache.drill.exec.store.parquet;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.vector.*;
-
import org.apache.drill.exec.vector.NullableBigIntVector;
import org.apache.drill.exec.vector.NullableFloat4Vector;
import org.apache.drill.exec.vector.NullableFloat8Vector;
import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.ValueVector;
+
import parquet.column.ColumnDescriptor;
import parquet.column.Encoding;
-import parquet.format.ConvertedType;
import parquet.format.SchemaElement;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.schema.PrimitiveType;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index 34287f3..0189c9b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -23,7 +23,6 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
-import com.google.common.collect.Maps;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
@@ -37,7 +36,14 @@ import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.RecordWriter;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
-import org.apache.drill.exec.store.dfs.*;
+import org.apache.drill.exec.store.dfs.BasicFormatMatcher;
+import org.apache.drill.exec.store.dfs.DrillPathFilter;
+import org.apache.drill.exec.store.dfs.FileSelection;
+import org.apache.drill.exec.store.dfs.FileSystemConfig;
+import org.apache.drill.exec.store.dfs.FormatMatcher;
+import org.apache.drill.exec.store.dfs.FormatPlugin;
+import org.apache.drill.exec.store.dfs.FormatSelection;
+import org.apache.drill.exec.store.dfs.MagicString;
import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
import org.apache.drill.exec.store.mock.MockStorageEngine;
import org.apache.hadoop.conf.Configuration;
@@ -52,6 +58,7 @@ import parquet.hadoop.ParquetFileWriter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
public class ParquetFormatPlugin implements FormatPlugin{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index 70560e7..703ad1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -24,8 +24,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import com.google.common.base.Preconditions;
-
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
@@ -36,7 +34,6 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
@@ -45,25 +42,39 @@ import org.apache.drill.exec.store.parquet.FixedByteAlignedReader.Decimal28Reade
import org.apache.drill.exec.store.parquet.FixedByteAlignedReader.Decimal38Reader;
import org.apache.drill.exec.store.parquet.NullableFixedByteAlignedReader.NullableDecimal28Reader;
import org.apache.drill.exec.store.parquet.NullableFixedByteAlignedReader.NullableDecimal38Reader;
-import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.Decimal28Column;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.Decimal38Column;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.NullableDecimal28Column;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.NullableDecimal38Column;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.NullableVarBinaryColumn;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.NullableVarCharColumn;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.NullableVarLengthColumn;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.VarBinaryColumn;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.VarCharColumn;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.VarLengthColumn;
+import org.apache.drill.exec.vector.Decimal28SparseVector;
+import org.apache.drill.exec.vector.Decimal38SparseVector;
+import org.apache.drill.exec.vector.NullableDecimal28SparseVector;
+import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
+import org.apache.drill.exec.vector.NullableVarBinaryVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarBinaryVector;
+import org.apache.drill.exec.vector.VarCharVector;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import parquet.column.ColumnDescriptor;
+import parquet.column.Encoding;
import parquet.format.ConvertedType;
import parquet.format.FileMetaData;
import parquet.format.SchemaElement;
import parquet.format.converter.ParquetMetadataConverter;
import parquet.hadoop.CodecFactoryExposer;
import parquet.hadoop.ParquetFileWriter;
-import parquet.column.Encoding;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.hadoop.metadata.ParquetMetadata;
import parquet.schema.PrimitiveType;
-
-import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.*;
-
-import com.google.common.base.Joiner;
import parquet.schema.PrimitiveType.PrimitiveTypeName;
public class ParquetRecordReader implements RecordReader {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
index 762942d..67bab5e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
@@ -17,8 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
-import com.fasterxml.jackson.annotation.*;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.FormatPluginConfig;
@@ -29,7 +28,12 @@ import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import java.io.IOException;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
@JsonTypeName("parquet-writer")
public class ParquetWriter extends AbstractWriter {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
index 2575c4d..813a799 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
@@ -17,12 +17,14 @@
*/
package org.apache.drill.exec.store.parquet;
-import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.*;
-import parquet.bytes.BytesUtils;
-
import java.io.IOException;
import java.util.List;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.NullableVarLengthColumn;
+import org.apache.drill.exec.store.parquet.VarLengthColumnReaders.VarLengthColumn;
+
+import parquet.bytes.BytesUtils;
+
public class VarLenBinaryReader {
ParquetRecordReader parentReader;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLengthColumnReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLengthColumnReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLengthColumnReaders.java
index fe9e574..56f687c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLengthColumnReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLengthColumnReaders.java
@@ -17,25 +17,28 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet;
+import java.math.BigDecimal;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
-import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder;
-import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.Decimal28SparseVector;
+import org.apache.drill.exec.vector.Decimal38SparseVector;
+import org.apache.drill.exec.vector.NullableDecimal28SparseVector;
+import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
+
import parquet.column.ColumnDescriptor;
-import parquet.format.ConvertedType;
import parquet.column.Encoding;
import parquet.format.SchemaElement;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.io.api.Binary;
-import java.math.BigDecimal;
-
public class VarLengthColumnReaders {
public static abstract class VarLengthColumn<V extends ValueVector> extends ColumnReader {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
index b32b601..435bd0d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
@@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.drill.exec.store.sys.PStore;
-import com.google.hive12.common.collect.Maps;
+import com.google.common.collect.Maps;
public class NoWriteLocalPStore<V> implements PStore<V>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NoWriteLocalPStore.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
index 8f788af..5350662 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
@@ -21,68 +21,14 @@
package org.apache.drill.exec.vector.complex.impl;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.ObjectArrays;
-import com.google.common.base.Charsets;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.*;
-import org.apache.drill.exec.vector.*;
-import org.apache.drill.exec.expr.holders.*;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.types.TypeProtos.*;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.vector.complex.*;
-import org.apache.drill.exec.vector.complex.reader.*;
-import org.apache.drill.exec.vector.complex.writer.*;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-import com.sun.codemodel.JType;
-import com.sun.codemodel.JCodeModel;
-
-import java.util.Arrays;
-import java.util.Random;
-import java.util.List;
-import java.io.Closeable;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-import org.apache.hadoop.io.Text;
-
-
-
-
-
-
-
-
-
-
-
-
-
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.vector.complex.MapVector;
+import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
index 2158fcc..41450ce 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
@@ -21,65 +21,13 @@
package org.apache.drill.exec.vector.complex.impl;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.ObjectArrays;
-import com.google.common.base.Charsets;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.*;
-import org.apache.drill.exec.vector.*;
-import org.apache.drill.exec.expr.holders.*;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.types.TypeProtos.*;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.vector.complex.*;
-import org.apache.drill.exec.vector.complex.reader.*;
-import org.apache.drill.exec.vector.complex.writer.*;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-import com.sun.codemodel.JType;
-import com.sun.codemodel.JCodeModel;
-
-import java.util.Arrays;
-import java.util.Random;
-import java.util.List;
-import java.io.Closeable;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-import org.apache.hadoop.io.Text;
-
-
-
-
-
-
-
-
-
-
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.MapVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
index 2666233..82da483 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.cache;
-import com.google.common.collect.Lists;
+import java.util.List;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
@@ -26,7 +26,11 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
@@ -41,7 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
-import java.util.List;
+import com.google.common.collect.Lists;
public class TestWriteToDisk extends ExecTest{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
index 6d7da12..a9c244a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.fn.impl;
+import java.util.Random;
+
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -24,11 +26,10 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
-import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
import org.apache.drill.exec.record.RecordBatch;
-import java.util.Random;
-
public class GeneratorFunctions extends ExecTest {
public static final Random random = new Random(1234L);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
index db026b6..ff45bb5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
@@ -17,25 +17,25 @@
*/
package org.apache.drill.exec.fn.impl;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
-import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.BigIntVector;
-import org.junit.Test;
import org.junit.Ignore;
+import org.junit.Test;
-import java.util.List;
-
-import static org.junit.Assert.*;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
public class TestDateFunctions extends PopUnitTestBase {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestDateFunctions.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
index 987661c..104923b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
@@ -18,7 +18,8 @@
package org.apache.drill.exec.fn.impl;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import mockit.Injectable;
import mockit.NonStrictExpectations;
@@ -28,28 +29,25 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.proto.BitControl;
import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.vector.BigIntVector;
-import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.Float8Vector;
+import org.apache.drill.exec.vector.IntVector;
import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
public class TestMathFunctions extends ExecTest {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
index ab00498..12c1c03 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
@@ -17,27 +17,25 @@
*/
package org.apache.drill.exec.physical.impl;
-import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
-import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.NullableBigIntVector;
-
import org.junit.Ignore;
import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import java.util.List;
-
/* This class tests the existing date types. Simply using date types
* by casting from VarChar, performing basic functions and converting
* back to VarChar.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 7ea01a6..5dd64e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -17,19 +17,16 @@
*/
package org.apache.drill.exec.physical.impl;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
import java.nio.charset.Charset;
import java.util.List;
-import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
-import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
@@ -37,10 +34,8 @@ import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.ValueVector;
-import org.junit.Assert;
import org.junit.Test;
-import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 2443ced..27a6caa 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -20,13 +20,20 @@ package org.apache.drill.exec.physical.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import com.sun.codemodel.JClassAlreadyExistsException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.common.util.FileUtils;
@@ -40,24 +47,20 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.resolver.FunctionResolver;
import org.apache.drill.exec.resolver.FunctionResolverFactory;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.junit.After;
import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import com.sun.codemodel.JClassAlreadyExistsException;
public class TestSimpleFunctions extends ExecTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
index 46ace78..3669c5d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
@@ -17,38 +17,35 @@
*/
package org.apache.drill.exec.physical.impl.agg;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.IntVector;
-import org.junit.AfterClass;
import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
public class TestAgg extends ExecTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAgg.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
index 6eb468c..e887584 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
@@ -17,38 +17,28 @@
*/
package org.apache.drill.exec.physical.impl.common;
-import static org.junit.Assert.*;
-import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.NamedExpression;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.expr.holders.IntHolder;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.vector.BigIntVector;
-import org.apache.drill.exec.vector.IntVector;
-import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
public class TestHashTable extends ExecTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHashTable.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
index 23bc28f..1f87371 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.physical.impl.filter;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import mockit.Injectable;
import mockit.NonStrictExpectations;
@@ -25,29 +26,23 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.vector.ValueVector;
-import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
public class TestSimpleFilter extends ExecTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFilter.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
index b2c5b19..abe3410 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
@@ -17,34 +17,31 @@
*/
package org.apache.drill.exec.physical.impl.trace;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
-import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
/*
* This test uses a physical plan with the mock scan that generates 100k records.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
index 998851a..5817e0b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
@@ -17,27 +17,26 @@
*/
package org.apache.drill.exec.physical.impl.trace;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.cache.VectorAccessibleSerializable;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.record.VectorAccessible;
@@ -47,12 +46,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
import org.junit.Test;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import com.codahale.metrics.MetricRegistry;
/*
* This test uses a simple physical plan with a mock-scan that
[17/32] git commit: More stats fixes: In PlanningSet,
get majorFragmentId from the physical plan.
Posted by ja...@apache.org.
More stats fixes: In PlanningSet, get majorFragmentId from the physical plan.
This is so the majorFragmentIds in the profiles will match those in the physical plan.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2b5ae075
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2b5ae075
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2b5ae075
Branch: refs/heads/master
Commit: 2b5ae075f75f51b81fa39b14c6a96090c2673ba0
Parents: 57a86d4
Author: Steven Phillips <sp...@maprtech.com>
Authored: Sat Jun 14 14:10:06 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:04:44 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/planner/fragment/PlanningSet.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2b5ae075/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
index 6795588..3164228 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
@@ -29,8 +29,7 @@ public class PlanningSet implements Iterable<Wrapper>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PlanningSet.class);
private Map<Fragment, Wrapper> fragmentMap = Maps.newHashMap();
- private int majorFragmentIdIndex = 0;
-
+
PlanningSet(){
}
@@ -45,7 +44,8 @@ public class PlanningSet implements Iterable<Wrapper>{
Wrapper get(Fragment node){
Wrapper wrapper = fragmentMap.get(node);
if(wrapper == null){
- wrapper = new Wrapper(node, majorFragmentIdIndex++);
+ int majorFragmentId = node.getRoot().getOperatorId() == 0 ? 0 : node.getSendingExchange().getChild().getOperatorId() >> 16;
+ wrapper = new Wrapper(node, majorFragmentId);
fragmentMap.put(node, wrapper);
}
return wrapper;
[06/32] git commit: DRILL-957: Fix schema failures when querying a
partitioned Avro table in Hive.
Posted by ja...@apache.org.
DRILL-957: Fix schema failures when querying a partitioned Avro table in Hive.
When initializing SerDe in HiveRecordReader, we are not passing the properties
from Table to SerDe. As schema is in table properties, Avro SerDe fails to find
the schema and throws errors. For non-partitioned tables this is ok, as the
MetaStoreUtils.getTableMetadata already adds Table properties, but not for
partitioned Avro tables. Fix is to pass table properties to SerDe if the
table is partitioned.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/aaaca6a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/aaaca6a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/aaaca6a7
Branch: refs/heads/master
Commit: aaaca6a7829733a120018e1388c68672d035820b
Parents: 3bcead7
Author: vkorukanti <ve...@gmail.com>
Authored: Sat Jun 14 18:32:18 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:53:13 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/store/hive/HiveRecordReader.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/aaaca6a7/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index ac0f036..c062f8c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.sql.Timestamp;
import java.sql.Date;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.drill.common.exceptions.DrillRuntimeException;
@@ -119,6 +120,15 @@ public class HiveRecordReader implements RecordReader {
JobConf job = new JobConf();
if (partition != null) {
properties = MetaStoreUtils.getPartitionMetadata(partition, table);
+
+ // SerDe expects properties from Table, but above call doesn't add Table properties.
+ // Include Table properties in final list in order to not to break SerDes that depend on
+ // Table properties. For example AvroSerDe gets the schema from properties (passed as second argument)
+ for (Map.Entry<String, String> entry : table.getParameters().entrySet()) {
+ if (entry.getKey() != null && entry.getKey() != null) {
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ }
} else {
properties = MetaStoreUtils.getTableMetadata(table);
}
[20/32] git commit: DRILL-974: Windows sqlline.bat - Correctly handle
spaces in the path to the Drill directory. Previously,
if Drill was on a path such as for example,
C:\My Home Dir\Drill Work\apache-drill-1.0.0-m2-incubating-SNAPSHOT,
then sqlline.bat
Posted by ja...@apache.org.
DRILL-974: Windows sqlline.bat - Correctly handle spaces in the path to the Drill directory. Previously, if Drill was on a path such as for example, C:\My Home Dir\Drill Work\apache-drill-1.0.0-m2-incubating-SNAPSHOT, then sqlline.bat would not correctly parse this path and be unable to start.
original fix by David Kewley ( dkewley@maprtech.com )
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7821b51b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7821b51b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7821b51b
Branch: refs/heads/master
Commit: 7821b51bded2ed016d4926a280fb3638e767fc8c
Parents: ebe49ea
Author: Patrick Wong <pw...@maprtech.com>
Authored: Thu Jun 12 17:55:54 2014 +0000
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:06 2014 -0700
----------------------------------------------------------------------
distribution/src/resources/sqlline.bat | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7821b51b/distribution/src/resources/sqlline.bat
----------------------------------------------------------------------
diff --git a/distribution/src/resources/sqlline.bat b/distribution/src/resources/sqlline.bat
index 0103541..dde4608 100755
--- a/distribution/src/resources/sqlline.bat
+++ b/distribution/src/resources/sqlline.bat
@@ -79,7 +79,7 @@ rem ----
set DRILL_BIN_DIR=%~dp0
pushd %DRILL_BIN_DIR%..
-set DRILL_HOME=%cd%
+set DRILL_HOME=^"%cd%^"
popd
if "test%DRILL_CONF_DIR%" == "test" (
[10/32] git commit: DRILL-981: Display content of map object in the
same order of original JSON object.
Posted by ja...@apache.org.
DRILL-981: Display content of map object in the same order of original JSON object.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/bc0bb4d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/bc0bb4d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/bc0bb4d5
Branch: refs/heads/master
Commit: bc0bb4d5012a9d51cdcfa0002a7b8b8753f01775
Parents: c789f1c
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Fri Jun 13 09:01:27 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:58:16 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/util/JsonStringHashMap.java | 4 +++-
.../java/org/apache/drill/exec/vector/complex/MapVector.java | 2 +-
.../org/apache/drill/exec/vector/complex/RepeatedMapVector.java | 4 ++--
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/bc0bb4d5/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
index 4b264bf..507cb28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -20,15 +20,17 @@ package org.apache.drill.exec.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
+
import org.apache.hadoop.io.Text;
import java.util.HashMap;
+import java.util.LinkedHashMap;
/*
* Simple class that extends the regular java.util.HashMap but overrides the
* toString() method of the HashMap class to produce a JSON string instead
*/
-public class JsonStringHashMap<K, V> extends HashMap<K, V> {
+public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
private static ObjectMapper mapper;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/bc0bb4d5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 8ea2796..ed2ad8a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -53,7 +53,7 @@ public class MapVector extends AbstractContainerVector {
public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REQUIRED).build();
- final HashMap<String, ValueVector> vectors = Maps.newHashMap();
+ final HashMap<String, ValueVector> vectors = Maps.newLinkedHashMap();
private final Map<String, VectorWithOrdinal> vectorIds = Maps.newHashMap();
private final IntObjectOpenHashMap<ValueVector> vectorsById = new IntObjectOpenHashMap<>();
private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/bc0bb4d5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index f41cfda..f05ab1b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -55,7 +55,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REPEATED).build();
private final UInt4Vector offsets; // offsets to start of each record
- private final Map<String, ValueVector> vectors = Maps.newHashMap();
+ private final Map<String, ValueVector> vectors = Maps.newLinkedHashMap();
private final Map<String, VectorWithOrdinal> vectorIds = Maps.newHashMap();
private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
private final IntObjectOpenHashMap<ValueVector> vectorsById = new IntObjectOpenHashMap<>();
@@ -356,7 +356,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
List<Object> l = new JsonStringArrayList();
int end = offsets.getAccessor().get(index+1);
for(int i = offsets.getAccessor().get(index); i < end; i++){
- Map<String, Object> vv = Maps.newHashMap();
+ Map<String, Object> vv = Maps.newLinkedHashMap();
for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
ValueVector v = e.getValue();
String k = e.getKey();
[18/32] git commit: DRILL-585: Switch getObject in SQL accessors to
return java.sql.* types.
Posted by ja...@apache.org.
DRILL-585: Switch getObject in SQL accessors to return java.sql.* types.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0648c9e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0648c9e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0648c9e9
Branch: refs/heads/master
Commit: 0648c9e9fb3c6fa607e8b46f4062ff6453b6a634
Parents: 2b5ae07
Author: Cliff Buchanan <cb...@maprtech.com>
Authored: Mon Jun 16 08:14:13 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:43:55 2014 -0700
----------------------------------------------------------------------
.../src/main/codegen/includes/vv_imports.ftl | 1 +
.../main/codegen/templates/SqlAccessors.java | 22 ++++++++++++++++++++
.../vector/accessor/AbstractSqlAccessor.java | 2 +-
.../jdbc/test/TestAggregateFunctionsQuery.java | 11 ++++++----
4 files changed, 31 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0648c9e9/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 114ecbe..9e16116 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -22,6 +22,7 @@ import io.netty.buffer.ByteBufInputStream;
import org.apache.commons.lang3.ArrayUtils;
+import org.apache.drill.exec.expr.fn.impl.StringFunctionUtil;
import org.apache.drill.exec.memory.AccountingByteBuf;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0648c9e9/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
index b7207f9..d05b7fd 100644
--- a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
+++ b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
@@ -45,9 +45,11 @@ public class ${name}Accessor extends AbstractSqlAccessor{
this.ac = vector.getAccessor();
}
+ <#if minor.class != "TimeStampTZ" && minor.class != "TimeStamp" && minor.class != "Time" && minor.class != "Date">
public Object getObject(int index){
return ac.getObject(index);
}
+ </#if>
<#if type.major == "VarLen">
@@ -65,6 +67,10 @@ public class ${name}Accessor extends AbstractSqlAccessor{
<#switch minor.class>
<#case "VarBinary">
+ public String getString(int index) {
+ byte [] b = ac.get(index);
+ return StringFunctionUtil.toBinaryString(io.netty.buffer.Unpooled.wrappedBuffer(b), 0, b.length);
+ }
<#break>
<#case "VarChar">
@Override
@@ -98,6 +104,10 @@ public class ${name}Accessor extends AbstractSqlAccessor{
<#else>
<#if minor.class == "TimeStampTZ">
+ public Object getObject(int index) {
+ return getTimestamp(index);
+ }
+
@Override
public Timestamp getTimestamp(int index) {
return new Timestamp(ac.getObject(index).getMillis());
@@ -113,6 +123,10 @@ public class ${name}Accessor extends AbstractSqlAccessor{
return ac.getObject(index);
}
<#elseif minor.class == "Date">
+ public Object getObject(int index) {
+ return getDate(index);
+ }
+
@Override
public Date getDate(int index) {
org.joda.time.DateTime date = new org.joda.time.DateTime(ac.get(index), org.joda.time.DateTimeZone.UTC);
@@ -120,6 +134,10 @@ public class ${name}Accessor extends AbstractSqlAccessor{
return new Date(date.getMillis());
}
<#elseif minor.class == "TimeStamp">
+ public Object getObject(int index) {
+ return getTimestamp(index);
+ }
+
@Override
public Timestamp getTimestamp(int index) {
org.joda.time.DateTime date = new org.joda.time.DateTime(ac.get(index), org.joda.time.DateTimeZone.UTC);
@@ -127,6 +145,10 @@ public class ${name}Accessor extends AbstractSqlAccessor{
return new Timestamp(date.getMillis());
}
<#elseif minor.class == "Time">
+ public Object getObject(int index) {
+ return getTime(index);
+ }
+
@Override
public Time getTime(int index) {
org.joda.time.DateTime time = new org.joda.time.DateTime(ac.get(index), org.joda.time.DateTimeZone.UTC);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0648c9e9/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
index 93089e7..9884612 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
@@ -99,7 +99,7 @@ abstract class AbstractSqlAccessor implements SqlAccessor {
@Override
public String getString(int index) throws InvalidAccessException{
- throw new InvalidAccessException("String");
+ return getObject(index).toString();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0648c9e9/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
index 5c5a218..812ee40 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
@@ -19,6 +19,7 @@ package org.apache.drill.jdbc.test;
import java.nio.file.Paths;
import java.sql.Connection;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
@@ -29,6 +30,7 @@ import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTest;
+import org.joda.time.chrono.ISOChronology;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -36,11 +38,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
-import org.joda.time.DateTime;
-
import com.google.common.base.Function;
import com.google.common.base.Stopwatch;
+
public class TestAggregateFunctionsQuery {
public static final String WORKING_PATH;
@@ -55,8 +56,10 @@ public class TestAggregateFunctionsQuery {
" FROM `employee.json`");
- String t = new DateTime(1998, 1, 1, 0, 0, 0, 0).toString();
- String t1 = new DateTime(1993, 5, 1, 0, 0, 0, 0).toString();
+ String t = new Date(ISOChronology.getInstance().getDateTimeMillis(1998, 1, 1, 0)).toString();
+ String t1 = new Date(ISOChronology.getInstance().getDateTimeMillis(1993, 5, 1, 0)).toString();
+ //String t = new Date(1998, 1, 1).toString();
+ //String t1 = new Date(1993, 5, 1).toString();
String result = String.format("MAX_DATE="+ t + "; " + "MIN_DATE=" + t1 + "\n");
[12/32] git commit: DRILL-534: Re-enable TPCH04
Posted by ja...@apache.org.
DRILL-534: Re-enable TPCH04
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/989d3364
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/989d3364
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/989d3364
Branch: refs/heads/master
Commit: 989d336406237593434af939771f2f06637e378b
Parents: 817bea5
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Jun 12 11:57:20 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:02:35 2014 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/drill/TestTpchDistributed.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/989d3364/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
index c388b3b..1c5a7f9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
@@ -45,7 +45,6 @@ public class TestTpchDistributed extends BaseTestQuery{
}
@Test
- @Ignore
public void tpch04() throws Exception{
testDistributed("queries/tpch/04.sql");
}
[25/32] git commit: DRILL-933: Remove old physical operator cost &
size concepts, add automatic size-based parallelization
Posted by ja...@apache.org.
DRILL-933: Remove old physical operator cost & size concepts, add automatic size-based parallelization
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6f54ab01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6f54ab01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6f54ab01
Branch: refs/heads/master
Commit: 6f54ab01a87f1bcf07ac502ea57b874fe3ddd9ac
Parents: e9ab7aa
Author: Jacques Nadeau <ja...@apache.org>
Authored: Fri Jun 13 09:34:07 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:20 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/hbase/HBaseGroupScan.java | 19 +---
.../drill/exec/store/hbase/HBaseSubScan.java | 12 --
.../org/apache/drill/exec/ExecConstants.java | 60 ++++++++--
.../drill/exec/physical/OperatorCost.java | 74 ------------
.../apache/drill/exec/physical/RecordField.java | 60 ----------
.../drill/exec/physical/base/AbstractBase.java | 9 ++
.../exec/physical/base/AbstractExchange.java | 25 -----
.../exec/physical/base/AbstractGroupScan.java | 8 --
.../exec/physical/base/AbstractMultiple.java | 17 +--
.../exec/physical/base/AbstractReceiver.java | 9 +-
.../exec/physical/base/AbstractSender.java | 15 +--
.../exec/physical/base/AbstractSingle.java | 19 ++--
.../exec/physical/base/AbstractSubScan.java | 13 +--
.../drill/exec/physical/base/Exchange.java | 26 ++---
.../drill/exec/physical/base/ExchangeCost.java | 68 -----------
.../drill/exec/physical/base/GroupScan.java | 31 +----
.../exec/physical/base/PhysicalOperator.java | 22 +---
.../drill/exec/physical/base/ScanStats.java | 82 ++++++++++++++
.../apache/drill/exec/physical/base/Size.java | 48 --------
.../exec/physical/config/BroadcastSender.java | 18 +--
.../exec/physical/config/ExternalSort.java | 27 +----
.../drill/exec/physical/config/Filter.java | 12 --
.../exec/physical/config/HashAggregate.java | 30 -----
.../drill/exec/physical/config/HashJoinPOP.java | 26 +----
.../exec/physical/config/IteratorValidator.java | 7 --
.../drill/exec/physical/config/Limit.java | 15 +--
.../exec/physical/config/MergeJoinPOP.java | 15 +--
.../physical/config/MergingReceiverPOP.java | 14 ---
.../physical/config/OrderedPartitionSender.java | 15 +--
.../drill/exec/physical/config/Project.java | 13 ---
.../exec/physical/config/RandomReceiver.java | 18 ---
.../drill/exec/physical/config/Screen.java | 9 --
.../physical/config/SelectionVectorRemover.java | 12 --
.../exec/physical/config/SingleSender.java | 8 --
.../apache/drill/exec/physical/config/Sort.java | 19 ----
.../physical/config/StreamingAggregate.java | 13 ---
.../apache/drill/exec/physical/config/TopN.java | 28 +----
.../drill/exec/physical/config/Trace.java | 17 ---
.../drill/exec/physical/config/UnionAll.java | 11 --
.../exec/physical/config/UnionExchange.java | 7 +-
.../apache/drill/exec/planner/ExecPlanner.java | 34 ------
.../drill/exec/planner/SimpleExecPlanner.java | 59 ----------
.../planner/fragment/MaterializedFragment.java | 69 ------------
.../planner/fragment/SimpleParallelizer.java | 70 +++++-------
.../drill/exec/planner/fragment/Stats.java | 43 ++-----
.../exec/planner/fragment/StatsCollector.java | 6 +-
.../exec/planner/logical/DrillScanRel.java | 12 +-
.../exec/planner/physical/AggPruleBase.java | 6 +-
.../planner/physical/BroadcastExchangePrel.java | 4 +-
.../physical/ConvertCountToDirectScan.java | 4 +-
.../drill/exec/planner/physical/FilterPrel.java | 3 +-
.../exec/planner/physical/HashAggPrel.java | 4 +-
.../exec/planner/physical/HashJoinPrel.java | 4 +-
.../physical/HashToMergeExchangePrel.java | 4 +-
.../physical/HashToRandomExchangePrel.java | 4 +-
.../drill/exec/planner/physical/LimitPrel.java | 4 +-
.../exec/planner/physical/MergeJoinPrel.java | 4 +-
.../planner/physical/PhysicalPlanCreator.java | 12 +-
.../exec/planner/physical/PlannerSettings.java | 5 +
.../drill/exec/planner/physical/Prel.java | 1 -
.../exec/planner/physical/ProjectPrel.java | 4 +-
.../drill/exec/planner/physical/ScanPrel.java | 24 ++--
.../drill/exec/planner/physical/ScreenPrel.java | 4 +-
.../physical/SelectionVectorRemoverPrel.java | 4 +-
.../physical/SingleMergeExchangePrel.java | 4 +-
.../drill/exec/planner/physical/SortPrel.java | 4 +-
.../exec/planner/physical/StreamAggPrel.java | 3 +-
.../drill/exec/planner/physical/TopNPrel.java | 4 +-
.../exec/planner/physical/UnionAllPrel.java | 6 +-
.../planner/physical/UnionDistinctPrel.java | 6 +-
.../planner/physical/UnionExchangePrel.java | 3 +-
.../drill/exec/planner/physical/WriterPrel.java | 4 +-
.../physical/visitor/BasePrelVisitor.java | 7 ++
.../visitor/ExcessiveExchangeIdentifier.java | 112 +++++++++++++++++++
.../planner/physical/visitor/PrelVisitor.java | 3 +-
.../planner/sql/handlers/DefaultSqlHandler.java | 22 +++-
.../drill/exec/rpc/control/WorkEventBus.java | 4 +-
.../server/options/SystemOptionManager.java | 16 ++-
.../exec/store/dfs/easy/EasyGroupScan.java | 22 ++--
.../drill/exec/store/dfs/easy/EasyWriter.java | 9 +-
.../exec/store/direct/DirectGroupScan.java | 14 +--
.../drill/exec/store/hive/HiveReadEntry.java | 3 +-
.../apache/drill/exec/store/hive/HiveScan.java | 25 +++--
.../drill/exec/store/hive/HiveSubScan.java | 13 ---
.../exec/store/ischema/InfoSchemaGroupScan.java | 19 +---
.../drill/exec/store/mock/MockGroupScanPOP.java | 40 ++-----
.../drill/exec/store/mock/MockStorePOP.java | 7 --
.../drill/exec/store/mock/MockSubScanPOP.java | 12 --
.../exec/store/parquet/ParquetGroupScan.java | 32 +-----
.../exec/store/parquet/ParquetRowGroupScan.java | 13 ---
.../drill/exec/store/parquet/ParquetWriter.java | 7 --
.../drill/exec/store/sys/SystemTableScan.java | 14 +--
.../apache/drill/exec/work/foreman/Foreman.java | 59 ++++++++--
.../java/org/apache/drill/BaseTestQuery.java | 7 ++
.../org/apache/drill/TestTpchDistributed.java | 2 +-
.../drill/exec/pop/TestFragmentChecker.java | 3 +-
96 files changed, 570 insertions(+), 1268 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
index e5a5fcc..8e9ae18 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
@@ -37,11 +37,11 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
+import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec;
@@ -340,21 +340,12 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst
}
@Override
- public OperatorCost getCost() {
- //TODO Figure out how to properly calculate cost
- return new OperatorCost(regionsToScan.size(), // network
- 1, // disk
- 1, // memory
- 1); // cpu
- }
-
- @Override
- public Size getSize() {
- // TODO - this is wrong, need to populate correctly
+ public ScanStats getScanStats() {
+ //TODO: look at stats for this.
int rowCount = (hbaseScanSpec.getFilter() != null ? 5 : 10) * regionsToScan.size();
int avgColumnSize = 10;
int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size();
- return new Size(rowCount, numColumns*avgColumnSize);
+ return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
index fafe9c9..fbac159 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
@@ -24,11 +24,9 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.hadoop.hbase.filter.Filter;
@@ -87,16 +85,6 @@ public class HBaseSubScan extends AbstractBase implements SubScan {
}
@Override
- public OperatorCost getCost() {
- return null;
- }
-
- @Override
- public Size getSize() {
- return null;
- }
-
- @Override
public boolean isExecutable() {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index b622eaa..6673c4c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -18,7 +18,10 @@
package org.apache.drill.exec;
import org.apache.drill.exec.server.options.OptionValidator;
+import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
+import org.apache.drill.exec.server.options.TypeValidators.DoubleValidator;
import org.apache.drill.exec.server.options.TypeValidators.LongValidator;
+import org.apache.drill.exec.server.options.TypeValidators.PositiveLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.StringValidator;
public interface ExecConstants {
@@ -41,10 +44,6 @@ public interface ExecConstants {
public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.exec.metrics.jmx.enabled";
public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.exec.metrics.log.enabled";
public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.exec.metrics.log.interval";
- public static final String GLOBAL_MAX_WIDTH = "drill.exec.work.global.max.width";
- public static final String MAX_WIDTH_PER_ENDPOINT = "drill.exec.work.max.width.per.endpoint";
- public static final String EXECUTOR_THREADS = "drill.exec.work.executor.threads";
- public static final String AFFINITY_FACTOR = "drill.exec.work.affinity.factor";
public static final String CLIENT_RPC_THREADS = "drill.exec.rpc.user.client.threads";
public static final String BIT_SERVER_RPC_THREADS = "drill.exec.rpc.bit.server.threads";
public static final String USER_SERVER_RPC_THREADS = "drill.exec.rpc.user.server.threads";
@@ -68,14 +67,59 @@ public interface ExecConstants {
public static final String FILESYSTEM_PARTITION_COLUMN_LABEL = "drill.exec.storage.file.partition.column.label";
public static final String HAZELCAST_SUBNETS = "drill.exec.cache.hazel.subnets";
public static final String TOP_LEVEL_MAX_ALLOC = "drill.exec.memory.top.max";
- public static final String OUTPUT_FORMAT_OPTION = "store.format";
- public static final OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION, "parquet");
- public static final String PARQUET_BLOCK_SIZE = "parquet.block.size";
- public static final OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new LongValidator(PARQUET_BLOCK_SIZE, 512*1024*1024);
public static final String HTTP_ENABLE = "drill.exec.http.enabled";
public static final String HTTP_PORT = "drill.exec.http.port";
public static final String SYS_STORE_PROVIDER_CLASS = "drill.exec.sys.store.provider.class";
public static final String SYS_STORE_PROVIDER_LOCAL_PATH = "drill.exec.sys.store.provider.local.path";
public static final String SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE = "drill.exec.sys.store.provider.local.write";
public static final String ERROR_ON_MEMORY_LEAK = "drill.exec.debug.error_on_leak";
+
+
+
+ public static final String OUTPUT_FORMAT_OPTION = "store.format";
+ public static final OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION, "parquet");
+ public static final String PARQUET_BLOCK_SIZE = "store.parquet.block-size";
+ public static final OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new LongValidator(PARQUET_BLOCK_SIZE, 512*1024*1024);
+
+
+
+
+ public static final String SLICE_TARGET = "planner.slice_target";
+ public static final OptionValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET, Long.MAX_VALUE, 1000000);
+
+ /**
+ * Limits the maximum level of parallelization to this factor time the number of Drillbits
+ */
+ public static final String MAX_WIDTH_PER_NODE_KEY = "planner.width.max_per_node";
+ public static final OptionValidator MAX_WIDTH_PER_NODE = new PositiveLongValidator(MAX_WIDTH_PER_NODE_KEY, Integer.MAX_VALUE, (long) Math.ceil(Runtime.getRuntime().availableProcessors() * 0.70));
+
+ /**
+ * The maximum level or parallelization any stage of the query can do. Note that while this
+ * might be the number of active Drillbits, realistically, this could be well beyond that
+ * number of we want to do things like speed results return.
+ */
+ public static final String MAX_WIDTH_GLOBAL_KEY = "planner.width.max_per_query";
+ public static final OptionValidator MAX_WIDTH_GLOBAL = new PositiveLongValidator(MAX_WIDTH_GLOBAL_KEY, Integer.MAX_VALUE, 1000);
+
+ /**
+ * Factor by which a node with endpoint affinity will be favored while creating assignment
+ */
+ public static final String AFFINITY_FACTOR_KEY = "planner.affinity_factor";
+ public static final OptionValidator AFFINITY_FACTOR = new DoubleValidator(AFFINITY_FACTOR_KEY, 1.2d);
+
+ public static final String ENABLE_QUEUE_KEY = "exec.queue.enable";
+ public static final OptionValidator ENABLE_QUEUE = new BooleanValidator(ENABLE_QUEUE_KEY, false);
+
+ public static final String LARGE_QUEUE_KEY = "exec.queue.large";
+ public static final OptionValidator LARGE_QUEUE_SIZE = new PositiveLongValidator(LARGE_QUEUE_KEY, 1000, 10);
+
+ public static final String SMALL_QUEUE_KEY = "exec.queue.small";
+ public static final OptionValidator SMALL_QUEUE_SIZE = new PositiveLongValidator(SMALL_QUEUE_KEY, 100000, 100);
+
+ public static final String QUEUE_THRESHOLD_KEY = "exec.queue.threshold";
+ public static final OptionValidator QUEUE_THRESHOLD_SIZE = new PositiveLongValidator(QUEUE_THRESHOLD_KEY, Long.MAX_VALUE, 30000000);
+
+ public static final String QUEUE_TIMEOUT_KEY = "exec.queue.timeout_millis";
+ public static final OptionValidator QUEUE_TIMEOUT = new PositiveLongValidator(QUEUE_TIMEOUT_KEY, Long.MAX_VALUE, 60*1000*5);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
deleted file mode 100644
index cc474b5..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class OperatorCost {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorCost.class);
-
- private final float network;
- private final float disk;
- private final float memory;
- private final float cpu;
-
-
-
- @JsonCreator
- public OperatorCost(@JsonProperty("network") float network, @JsonProperty("disk") float disk, @JsonProperty("memory") float memory, @JsonProperty("cpu") float cpu) {
- super();
- this.network = network;
- this.disk = disk;
- this.memory = memory;
- this.cpu = cpu;
- }
-
- public float getNetwork() {
- return network;
- }
-
- public float getDisk() {
- return disk;
- }
-
- public float getMemory() {
- return memory;
- }
-
- public float getCpu() {
- return cpu;
- }
-
- public static OperatorCost combine(OperatorCost c1, OperatorCost c2){
- return new OperatorCost(c1.network + c2.network, c1.disk + c2.disk, c1.memory + c2.memory, c1.cpu + c2.cpu);
- }
-
- public OperatorCost add(OperatorCost c2){
- return combine(this, c2);
- }
-
- public int compare(OperatorCost c2) {
- float thisTotal = this.network + this.disk + this.memory + this.cpu;
- float c2Total = c2.network + c2.disk + c2.memory + c2.cpu;
- if (thisTotal < c2Total) return -1;
- else if (thisTotal == c2Total) return 0;
- else return 1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/RecordField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/RecordField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/RecordField.java
deleted file mode 100644
index 8f3dd2e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/RecordField.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class RecordField {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordField.class);
-
-
- private MajorType type;
- private ValueMode mode;
-
- @JsonCreator
- public RecordField(@JsonProperty("type") MajorType type, @JsonProperty("mode") ValueMode mode) {
- super();
- this.type = type;
- this.mode = mode;
- }
-
- public MajorType getType() {
- return type;
- }
-
- public ValueMode getMode() {
- return mode;
- }
-
- public static enum ValueMode {
- VECTOR,
- DICT,
- RLE
- }
-
- public static enum ValueType {
- OPTIONAL,
- REQUIRED,
- REPEATED
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractBase.java
index a028252..54f9ef8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractBase.java
@@ -29,6 +29,7 @@ public abstract class AbstractBase implements PhysicalOperator{
protected long initialAllocation = 1000000L;
protected long maxAllocation = 10000000000L;
private int id;
+ private double cost;
@Override
public void accept(GraphVisitor<PhysicalOperator> visitor) {
@@ -65,6 +66,14 @@ public abstract class AbstractBase implements PhysicalOperator{
return initialAllocation;
}
+ public double getCost() {
+ return cost;
+ }
+
+ public void setCost(double cost) {
+ this.cost = cost;
+ }
+
@Override
public long getMaxAllocation() {
return maxAllocation;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractExchange.java
index 4a918b4..8009ede 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractExchange.java
@@ -20,12 +20,8 @@ package org.apache.drill.exec.physical.base;
import java.util.List;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
public abstract class AbstractExchange extends AbstractSingle implements Exchange {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractExchange.class);
@@ -62,32 +58,11 @@ public abstract class AbstractExchange extends AbstractSingle implements Exchang
}
@Override
- public OperatorCost getAggregateSendCost() {
- return getExchangeCost().getSendCost();
- }
-
- @Override
- public OperatorCost getAggregateReceiveCost() {
- return getExchangeCost().getReceiveCost();
- }
-
- @Override
public final <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitExchange(this, value);
}
@Override
- public ExchangeCost getExchangeCost(){
- return ExchangeCost.getSimpleEstimate(getSize());
- }
-
- @JsonIgnore
- @Override
- public OperatorCost getCost() {
- return getExchangeCost().getCombinedCost();
- }
-
- @Override
public int getOperatorType() {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
index 4978450..c9b8e80 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
@@ -67,14 +67,6 @@ public abstract class AbstractGroupScan extends AbstractBase implements GroupSca
}
/**
- * By default, groupscan does not have exact row count, does not have exact column value count.
- */
- @Override
- public GroupScanProperty getProperty() {
- return GroupScanProperty.NO_EXACT_ROW_COUNT;
- }
-
- /**
* By default, throw exception, since group scan does not have exact column value count.
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractMultiple.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractMultiple.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractMultiple.java
index 6b774ed..7326be6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractMultiple.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractMultiple.java
@@ -25,8 +25,8 @@ import java.util.Iterator;
* Describes an operator that expects more than one children operators as its input.
*/
public abstract class AbstractMultiple extends AbstractBase{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractMultiple.class);
-
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractMultiple.class);
+
protected final PhysicalOperator[] children;
protected AbstractMultiple(PhysicalOperator[] children) {
@@ -36,18 +36,11 @@ public abstract class AbstractMultiple extends AbstractBase{
public PhysicalOperator[] getChildren() {
return children;
}
-
+
@Override
public Iterator<PhysicalOperator> iterator() {
return Iterators.forArray(children);
}
-
- public Size getSize() {
- Size size = new Size(0,0);
- for(PhysicalOperator child:children){
- size.add(child.getSize());
- }
- return size;
- }
-
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractReceiver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractReceiver.java
index 2a9cab7..f621a26 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractReceiver.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractReceiver.java
@@ -20,9 +20,6 @@ package org.apache.drill.exec.physical.base;
import java.util.Iterator;
import java.util.List;
-import org.apache.drill.exec.physical.OperatorCost;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
@@ -31,12 +28,12 @@ public abstract class AbstractReceiver extends AbstractBase implements Receiver{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractReceiver.class);
- private final int oppositeMajorFragmentId;
-
+ private final int oppositeMajorFragmentId;
+
public AbstractReceiver(int oppositeMajorFragmentId){
this.oppositeMajorFragmentId = oppositeMajorFragmentId;
}
-
+
@Override
public Iterator<PhysicalOperator> iterator() {
return Iterators.emptyIterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSender.java
index ae6643d..53a0721 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSender.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSender.java
@@ -17,9 +17,6 @@
*/
package org.apache.drill.exec.physical.base;
-import org.apache.drill.exec.physical.OperatorCost;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
@@ -27,7 +24,7 @@ public abstract class AbstractSender extends AbstractSingle implements Sender {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSender.class);
protected final int oppositeMajorFragmentId;
-
+
public AbstractSender(int oppositeMajorFragmentId, PhysicalOperator child) {
super(child);
this.oppositeMajorFragmentId = oppositeMajorFragmentId;
@@ -39,15 +36,9 @@ public abstract class AbstractSender extends AbstractSingle implements Sender {
}
@Override
- public OperatorCost getCost() {
- float network = child.getSize().getAggSize();
- return new OperatorCost(network, 0, 1000, child.getSize().getRecordCount());
- }
-
- @Override
public int getOppositeMajorFragmentId() {
return oppositeMajorFragmentId;
}
-
-
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSingle.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSingle.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSingle.java
index 4b6499f..6c0b98f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSingle.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSingle.java
@@ -31,7 +31,7 @@ import com.google.common.collect.Iterators;
*/
public abstract class AbstractSingle extends AbstractBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSingle.class);
-
+
protected final PhysicalOperator child;
public AbstractSingle(PhysicalOperator child) {
@@ -47,22 +47,17 @@ public abstract class AbstractSingle extends AbstractBase{
public PhysicalOperator getChild(){
return child;
}
-
- @Override
- public Size getSize() {
- return child.getSize();
- }
-
+
@Override
@JsonIgnore
public final PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
Preconditions.checkArgument(children.size() == 1);
return getNewWithChild(children.iterator().next());
}
-
+
protected abstract PhysicalOperator getNewWithChild(PhysicalOperator child);
-
-
-
-
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
index 97334ea..a36a46e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.graph.GraphVisitor;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import com.google.common.collect.Iterators;
@@ -31,16 +30,6 @@ public abstract class AbstractSubScan extends AbstractBase implements SubScan{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSubScan.class);
@Override
- public OperatorCost getCost() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Size getSize() {
- throw new UnsupportedOperationException();
- }
-
- @Override
public boolean isExecutable() {
return true;
}
@@ -67,7 +56,7 @@ public abstract class AbstractSubScan extends AbstractBase implements SubScan{
public Iterator<PhysicalOperator> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
public SelectionVectorMode getSVMode() {
return SelectionVectorMode.NONE;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Exchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Exchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Exchange.java
index 2d2372a..afbdac9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Exchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Exchange.java
@@ -20,28 +20,16 @@ package org.apache.drill.exec.physical.base;
import java.util.List;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
public interface Exchange extends PhysicalOperator {
- @JsonIgnore
- public abstract OperatorCost getAggregateSendCost();
-
- @JsonIgnore
- public abstract OperatorCost getAggregateReceiveCost();
-
- @JsonIgnore
- public abstract ExchangeCost getExchangeCost();
-
/**
* Inform this Exchange node about its sender locations. This list should be index-ordered the same as the expected
* minorFragmentIds for each sender.
- *
+ *
* @param senderLocations
*/
public abstract void setupSenders(int majorFragmentId, List<DrillbitEndpoint> senderLocations) throws PhysicalOperatorSetupException;
@@ -49,7 +37,7 @@ public interface Exchange extends PhysicalOperator {
/**
* Inform this Exchange node about its receiver locations. This list should be index-ordered the same as the expected
* minorFragmentIds for each receiver.
- *
+ *
* @param receiverLocations
*/
public abstract void setupReceivers(int majorFragmentId, List<DrillbitEndpoint> receiverLocations) throws PhysicalOperatorSetupException;
@@ -57,7 +45,7 @@ public interface Exchange extends PhysicalOperator {
/**
* Get the Sender associated with the given minorFragmentId. Cannot be called until after setupSenders() and
* setupReceivers() have been called.
- *
+ *
* @param minorFragmentId
* The minor fragment id, must be in the range [0, fragment.width).
* @param child
@@ -69,7 +57,7 @@ public interface Exchange extends PhysicalOperator {
/**
* Get the Receiver associated with the given minorFragmentId. Cannot be called until after setupSenders() and
* setupReceivers() have been called.
- *
+ *
* @param minorFragmentId
* The minor fragment id, must be in the range [0, fragment.width).
* @return The materialized recevier for the given arguments.
@@ -78,7 +66,7 @@ public interface Exchange extends PhysicalOperator {
/**
* The widest width this sender can send (max sending parallelization). Typically Integer.MAX_VALUE.
- *
+ *
* @return
*/
@JsonIgnore
@@ -86,9 +74,9 @@ public interface Exchange extends PhysicalOperator {
/**
* Return the feeding child of this operator node.
- *
+ *
* @return
*/
public PhysicalOperator getChild();
-
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ExchangeCost.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ExchangeCost.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ExchangeCost.java
deleted file mode 100644
index 8158a2e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ExchangeCost.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.base;
-
-import org.apache.drill.exec.physical.OperatorCost;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * A container class that holds both send and receive costs for an exchange node.
- */
-public class ExchangeCost {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExchangeCost.class);
-
- private final OperatorCost send;
- private final OperatorCost receive;
- private final OperatorCost combined;
-
- @JsonCreator
- public ExchangeCost(@JsonProperty("send") OperatorCost send, @JsonProperty("receive") OperatorCost receive) {
- this.send = send;
- this.receive = receive;
- this.combined = OperatorCost.combine(send, receive);
- }
-
-
-
- @JsonIgnore
- public OperatorCost getCombinedCost(){
- return combined;
- }
-
- @JsonProperty("send")
- public OperatorCost getSendCost() {
- return send;
- }
-
- @JsonProperty("receive")
- public OperatorCost getReceiveCost() {
- return receive;
- }
-
- public static ExchangeCost getSimpleEstimate(Size s){
- long cnt = s.getRecordCount();
- int sz = s.getRecordSize();
- OperatorCost send = new OperatorCost(cnt*sz, 0, 0, cnt);
- OperatorCost receive = new OperatorCost(cnt*sz, 0, 0, cnt);
- return new ExchangeCost(send, receive);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
index 3412c6b..fa20a90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
@@ -46,6 +46,9 @@ public interface GroupScan extends Scan, HasAffinity{
@JsonIgnore
public abstract String getDigest();
+ @JsonIgnore
+ public ScanStats getScanStats();
+
/**
* Returns a clone of GroupScan instance, except that the new GroupScan will use the provided list of columns .
*/
@@ -62,32 +65,4 @@ public interface GroupScan extends Scan, HasAffinity{
*/
public long getColumnValueCount(SchemaPath column);
- /**
- * Return if GroupScan knows the exact row count in the result of getSize() call.
- * By default, groupscan does not know the exact row count, before it scans every rows.
- * Currently, parquet group scan will return the exact row count.
- */
- @JsonIgnore
- public GroupScanProperty getProperty();
-
- public class GroupScanProperty {
- public static final GroupScanProperty NO_EXACT_ROW_COUNT = new GroupScanProperty(false, false);
- public static final GroupScanProperty EXACT_ROW_COUNT = new GroupScanProperty(true, true);
-
- private boolean hasExactRowCount, hasExactColumnValueCount;
-
- public GroupScanProperty (boolean hasExactRowCount, boolean hasExactColumnValueCount) {
- this.hasExactRowCount = hasExactRowCount;
- this.hasExactColumnValueCount = hasExactColumnValueCount;
- }
-
- public boolean hasExactRowCount() {
- return hasExactRowCount;
- }
-
- public boolean hasExactColumnValueCount() {
- return hasExactColumnValueCount;
- }
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
index 4dd2aef..8f51390 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.graph.GraphValue;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -41,21 +40,6 @@ import com.fasterxml.jackson.annotation.ObjectIdGenerators;
public interface PhysicalOperator extends GraphValue<PhysicalOperator> {
/**
- * Get the cost of execution of this particular operator.
- *
- * @return
- */
- @JsonIgnore
- public OperatorCost getCost();
-
- /**
- * Get the estimated size of this particular operator.
- * @return
- */
- @JsonIgnore
- public Size getSize();
-
- /**
* Describes whether or not a particular physical operator can actually be executed. Most physical operators can be
* executed. However, Exchange nodes cannot be executed. In order to be executed, they must be converted into their
* Exec sub components.
@@ -105,6 +89,12 @@ public interface PhysicalOperator extends GraphValue<PhysicalOperator> {
@JsonProperty("@id")
public void setOperatorId(int id);
+ @JsonProperty("cost")
+ public void setCost(double cost);
+
+ @JsonProperty("cost")
+ public double getCost();
+
@JsonIgnore
public int getOperatorType();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
new file mode 100644
index 0000000..ba36931
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.base;
+
+
+public class ScanStats {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanStats.class);
+
+
+ public static final ScanStats TRIVIAL_TABLE = new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, 20, 1, 1);
+
+ private final long recordCount;
+ private final float cpuCost;
+ private final float diskCost;
+ private final GroupScanProperty property;
+
+ public ScanStats(GroupScanProperty property, long recordCount, float cpuCost, float diskCost) {
+ super();
+ this.recordCount = recordCount;
+ this.cpuCost = cpuCost;
+ this.diskCost = diskCost;
+ this.property = property;
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public float getCpuCost() {
+ return cpuCost;
+ }
+
+ public float getDiskCost() {
+ return diskCost;
+ }
+
+ /**
+ * Return if GroupScan knows the exact row count in the result of getSize() call.
+ * By default, groupscan does not know the exact row count, before it scans every rows.
+ * Currently, parquet group scan will return the exact row count.
+ */
+ public GroupScanProperty getGroupScanProperty() {
+ return property;
+ }
+
+
+
+ public static enum GroupScanProperty {
+ NO_EXACT_ROW_COUNT(false, false),
+ EXACT_ROW_COUNT(true, true);
+
+ private boolean hasExactRowCount, hasExactColumnValueCount;
+
+ GroupScanProperty (boolean hasExactRowCount, boolean hasExactColumnValueCount) {
+ this.hasExactRowCount = hasExactRowCount;
+ this.hasExactColumnValueCount = hasExactColumnValueCount;
+ }
+
+ public boolean hasExactRowCount() {
+ return hasExactRowCount;
+ }
+
+ public boolean hasExactColumnValueCount() {
+ return hasExactColumnValueCount;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Size.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Size.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Size.java
deleted file mode 100644
index caf331f..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Size.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.base;
-
-public class Size {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Size.class);
-
- private final long rowCount;
- private final int rowSize;
-
- public Size(long rowCount, int rowSize) {
- super();
- this.rowCount = rowCount;
- this.rowSize = rowSize;
- }
-
- public long getRecordCount() {
- return rowCount;
- }
-
- public int getRecordSize() {
- return rowSize;
- }
-
- public Size add(Size s){
- return new Size(rowCount + s.rowCount, Math.max(rowSize, s.rowSize));
- }
-
- public long getAggSize(){
- return rowCount * rowSize;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastSender.java
index 88c40a1..1827367 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastSender.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastSender.java
@@ -18,19 +18,17 @@
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
+import java.util.List;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSender;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import java.util.List;
-
-import static org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("broadcast-sender")
public class BroadcastSender extends AbstractSender {
@@ -46,12 +44,6 @@ public class BroadcastSender extends AbstractSender {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(child.getSize().getAggSize() * destinations.size(),
- 0, 1000, child.getSize().getRecordCount());
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new BroadcastSender(oppositeMajorFragmentId, child, destinations);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ExternalSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ExternalSort.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ExternalSort.java
index a9e38ab..8af69fa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ExternalSort.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/ExternalSort.java
@@ -17,18 +17,16 @@
*/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
+import java.util.List;
import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("external-sort")
public class ExternalSort extends Sort {
@@ -53,23 +51,6 @@ public class ExternalSort extends Sort {
}
@Override
- public OperatorCost getCost() {
- Size childSize = child.getSize();
- long n = childSize.getRecordCount();
- long width = childSize.getRecordSize();
-
- //TODO: Magic Number, let's assume 1/10 of data can fit in memory.
- int k = 10;
- long n2 = n/k;
- double cpuCost =
- k * n2 * (Math.log(n2)/Math.log(2)) + //
- n * (Math.log(k)/Math.log(2));
- double diskCost = n*width*2;
-
- return new OperatorCost(0, (float) diskCost, (float) n2*width, (float) cpuCost);
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new ExternalSort(child, orderings, reverse);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Filter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Filter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Filter.java
index 99b5f46..7a253d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Filter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Filter.java
@@ -18,11 +18,9 @@
package org.apache.drill.exec.physical.config;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -55,21 +53,11 @@ public class Filter extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
- return child.getCost();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new Filter(child, expr, selectivity);
}
@Override
- public Size getSize() {
- return new Size( (long) (child.getSize().getRecordCount()*selectivity), child.getSize().getRecordSize());
- }
-
- @Override
public SelectionVectorMode getSVMode() {
if (child.getSVMode().equals(SelectionVectorMode.FOUR_BYTE)) {
return SelectionVectorMode.FOUR_BYTE;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
index d8fdc4d..e4ce5f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
@@ -18,11 +18,9 @@
package org.apache.drill.exec.physical.config;
import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.impl.common.HashTable;
import org.apache.drill.exec.physical.impl.common.HashTableConfig;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
@@ -82,39 +80,11 @@ public class HashAggregate extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
-
- final float hashCpuCost = (float) 10000.0; // temporarily set this high until we calibrate hashaggr vs. streaming-aggr costs.
- Size childSize = child.getSize();
- long n = childSize.getRecordCount();
- long width = childSize.getRecordSize();
- int numExprs = getGroupByExprs().length;
-
- double cpuCost = n * numExprs * hashCpuCost;
- double diskCost = 0; // for now assume hash table fits in memory
-
- return new OperatorCost(0, (float) diskCost, (float) n*width, (float) cpuCost);
- }
-
- public void logCostInfo(OperatorCost HACost, OperatorCost SACost) {
- logger.debug("HashAggregate cost: cpu = {}, disk = {}, memory = {}, network = {}.", HACost.getCpu(), HACost.getDisk(), HACost.getMemory(), HACost.getNetwork());
- logger.debug("Streaming aggregate cost: cpu = {}, disk = {}, memory = {}, network = {}.", SACost.getCpu(), SACost.getDisk(), SACost.getMemory(), SACost.getNetwork());
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new HashAggregate(child, groupByExprs, aggrExprs, cardinality);
}
@Override
- public Size getSize() {
- // not a great hack...
- return new Size( (long) (child.getSize().getRecordCount()*cardinality), child.getSize().getRecordSize());
- }
-
-
-
- @Override
public int getOperatorType() {
return CoreOperatorType.HASH_AGGREGATE_VALUE;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashJoinPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashJoinPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashJoinPOP.java
index 06c23f3..f9cae48 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashJoinPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashJoinPOP.java
@@ -18,30 +18,22 @@
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
import java.util.Iterator;
import java.util.List;
-import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.logical.data.JoinCondition;
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+import org.eigenbase.rel.JoinRelType;
-import com.google.common.collect.Lists;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
-
-import org.apache.drill.exec.physical.impl.common.HashTable;
-import org.apache.drill.exec.physical.impl.common.HashTableConfig;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import org.eigenbase.rel.JoinRelType;
+import com.google.common.collect.Lists;
@JsonTypeName("hash-join")
public class HashJoinPOP extends AbstractBase {
@@ -53,11 +45,6 @@ public class HashJoinPOP extends AbstractBase {
private final List<JoinCondition> conditions;
private final JoinRelType joinType;
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(0,0,0,0);
- }
-
@JsonCreator
public HashJoinPOP(
@JsonProperty("left") PhysicalOperator left,
@@ -73,11 +60,6 @@ public class HashJoinPOP extends AbstractBase {
}
@Override
- public Size getSize() {
- return left.getSize().add(right.getSize());
- }
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitHashJoin(this, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
index 94bffea..64cf7c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
@@ -17,11 +17,9 @@
*/
package org.apache.drill.exec.physical.config;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
public class IteratorValidator extends AbstractSingle{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IteratorValidator.class);
@@ -32,11 +30,6 @@ public class IteratorValidator extends AbstractSingle{
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(0f, 0f, 0f, 0f);
- }
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitIteratorValidator(this, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Limit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Limit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Limit.java
index 0db58ec..0038e4e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Limit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Limit.java
@@ -17,18 +17,16 @@
******************************************************************************/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
@JsonTypeName("limit")
public class Limit extends AbstractSingle {
private final Integer first;
@@ -55,11 +53,6 @@ public class Limit extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(0, 0, 0, 0.25f);
- }
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitLimit(this, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
index b79a07e..19f9b08 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
@@ -20,22 +20,19 @@ package org.apache.drill.exec.physical.config;
import java.util.Iterator;
import java.util.List;
-import org.apache.drill.common.logical.data.Join;
import org.apache.drill.common.logical.data.JoinCondition;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.eigenbase.rel.JoinRelType;
-import com.google.common.collect.Lists;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
@JsonTypeName("merge-join")
public class MergeJoinPOP extends AbstractBase{
@@ -47,11 +44,6 @@ public class MergeJoinPOP extends AbstractBase{
private final List<JoinCondition> conditions;
private final JoinRelType joinType;
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(0,0,0,0);
- }
-
@JsonCreator
public MergeJoinPOP(
@JsonProperty("left") PhysicalOperator left,
@@ -68,11 +60,6 @@ public class MergeJoinPOP extends AbstractBase{
}
@Override
- public Size getSize() {
- return left.getSize().add(right.getSize());
- }
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitMergeJoin(this, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
index da5e7e9..31d5945 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
@@ -20,10 +20,8 @@ package org.apache.drill.exec.physical.config;
import java.util.List;
import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractReceiver;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
@@ -63,22 +61,10 @@ public class MergingReceiverPOP extends AbstractReceiver{
}
@Override
- public OperatorCost getCost() {
- //TODO: deal with receiver cost through exchange.
- return new OperatorCost(1,1,1,1);
- }
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitMergingReceiver(this, value);
}
- @Override
- public Size getSize() {
- //TODO: deal with size info through exchange.
- return new Size(1,1);
- }
-
public List<Ordering> getOrderings() {
return orderings;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
index d73926e..0a2b9be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
@@ -21,18 +21,16 @@ import java.util.List;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSender;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import com.google.common.collect.Lists;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Lists;
@JsonTypeName("OrderedPartitionSender")
public class OrderedPartitionSender extends AbstractSender {
@@ -88,17 +86,6 @@ public class OrderedPartitionSender extends AbstractSender {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(0, 0, 1000, child.getSize().getRecordCount());
- }
-
- @Override
- public Size getSize() {
- //TODO: This should really change the row width...
- return child.getSize();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new OrderedPartitionSender(orderings, ref, child, endpoints, oppositeMajorFragmentId, sendingWidth, recordsToSample, samplingFactor,
completionFactor);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Project.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Project.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Project.java
index 8a3f5e2..7b58ecd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Project.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Project.java
@@ -20,11 +20,9 @@ package org.apache.drill.exec.physical.config;
import java.util.List;
import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -54,17 +52,6 @@ public class Project extends AbstractSingle{
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(0, 0, 1000, child.getSize().getRecordCount());
- }
-
- @Override
- public Size getSize() {
- //TODO: This should really change the row width...
- return child.getSize();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new Project(exprs, child);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
index f517f7c..b04db40 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/RandomReceiver.java
@@ -19,19 +19,14 @@ package org.apache.drill.exec.physical.config;
import java.util.List;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractReceiver;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Preconditions;
@JsonTypeName("random-receiver")
public class RandomReceiver extends AbstractReceiver{
@@ -58,24 +53,11 @@ public class RandomReceiver extends AbstractReceiver{
}
@Override
- public OperatorCost getCost() {
- //TODO: deal with receiver cost through exchange.
- return new OperatorCost(1,1,1,1);
- }
-
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitRandomReceiver(this, value);
}
@Override
- public Size getSize() {
- //TODO: deal with size info through exchange.
- return new Size(1,1);
- }
-
- @Override
public int getOperatorType() {
return CoreOperatorType.RANDOM_RECEIVER_VALUE;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Screen.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Screen.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Screen.java
index 4f5cb47..5f0648d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Screen.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Screen.java
@@ -22,12 +22,9 @@ import java.util.List;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractStore;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Root;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.Store;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
@@ -36,7 +33,6 @@ import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Preconditions;
@JsonTypeName("screen")
public class Screen extends AbstractStore {
@@ -90,11 +86,6 @@ public class Screen extends AbstractStore {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(0, 0, 1000, child.getSize().getRecordCount());
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new Screen(child, endpoint);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SelectionVectorRemover.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SelectionVectorRemover.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SelectionVectorRemover.java
index 13cbbe2..194c46f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SelectionVectorRemover.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SelectionVectorRemover.java
@@ -17,11 +17,9 @@
*/
package org.apache.drill.exec.physical.config;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -45,21 +43,11 @@ public class SelectionVectorRemover extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
- return child.getCost();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new SelectionVectorRemover(child);
}
@Override
- public Size getSize() {
- return new Size( (long) (child.getSize().getRecordCount()), child.getSize().getRecordSize());
- }
-
- @Override
public SelectionVectorMode getSVMode() {
return SelectionVectorMode.NONE;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleSender.java
index 82962ef..4a11a51 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleSender.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleSender.java
@@ -20,8 +20,6 @@ package org.apache.drill.exec.physical.config;
import java.util.Collections;
import java.util.List;
-import org.apache.drill.common.graph.GraphVisitor;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSender;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
@@ -55,12 +53,6 @@ public class SingleSender extends AbstractSender {
}
@Override
- public OperatorCost getCost() {
- long recordSize = child.getSize().getRecordSize() * child.getSize().getRecordCount();
- return new OperatorCost((float) recordSize, recordSize, 0, child.getSize().getRecordCount()/(1<<16));
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new SingleSender(oppositeMajorFragmentId, child, destination);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
index d7415d8..1aa53d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
@@ -20,11 +20,9 @@ package org.apache.drill.exec.physical.config;
import java.util.List;
import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -60,23 +58,6 @@ public class Sort extends AbstractSingle{
}
@Override
- public OperatorCost getCost() {
- Size childSize = child.getSize();
- long n = childSize.getRecordCount();
- long width = childSize.getRecordSize();
-
- //TODO: Magic Number, let's assume 1/10 of data can fit in memory.
- int k = 10;
- long n2 = n/k;
- double cpuCost =
- k * n2 * (Math.log(n2)/Math.log(2)) + //
- n * (Math.log(k)/Math.log(2));
- double diskCost = n*width*2;
-
- return new OperatorCost(0, (float) diskCost, (float) n2*width, (float) cpuCost);
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new Sort(child, orderings, reverse);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/StreamingAggregate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/StreamingAggregate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/StreamingAggregate.java
index 6e06c24..9486f6d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/StreamingAggregate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/StreamingAggregate.java
@@ -18,11 +18,9 @@
package org.apache.drill.exec.physical.config;
import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -61,22 +59,11 @@ public class StreamingAggregate extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
- return child.getCost();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new StreamingAggregate(child, keys, exprs, cardinality);
}
@Override
- public Size getSize() {
- // not a great hack...
- return new Size( (long) (child.getSize().getRecordCount()*cardinality), child.getSize().getRecordSize());
- }
-
- @Override
public int getOperatorType() {
return CoreOperatorType.STREAMING_AGGREGATE_VALUE;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
index dfd142a..4e3d9cd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/TopN.java
@@ -17,19 +17,16 @@
*/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
+import java.util.List;
import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("top-n")
public class TopN extends Sort {
@@ -61,23 +58,6 @@ public class TopN extends Sort {
}
@Override
- public OperatorCost getCost() {
- Size childSize = child.getSize();
- long n = childSize.getRecordCount();
- long width = childSize.getRecordSize();
-
- //TODO: Magic Number, let's assume 1/10 of data can fit in memory.
- int k = 10;
- long n2 = n/k;
- double cpuCost =
- k * n2 * (Math.log(n2)/Math.log(2)) + //
- n * (Math.log(k)/Math.log(2));
- double diskCost = n*width*2;
-
- return new OperatorCost(0, (float) diskCost, (float) n2*width, (float) cpuCost);
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new TopN(child, orderings, reverse, limit);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Trace.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Trace.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Trace.java
index f4ba842..fb24881 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Trace.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Trace.java
@@ -18,11 +18,9 @@
package org.apache.drill.exec.physical.config;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -50,21 +48,6 @@ public class Trace extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
-
- /* Compute the total size (row count * row size) */
- Size size = child.getSize();
- long diskSize = size.getRecordCount() * size.getRecordSize();
-
- return new OperatorCost(0, diskSize, 0, child.getSize().getRecordCount());
- }
-
- @Override
- public Size getSize() {
- return child.getSize();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new Trace(child, traceTag);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
index 06ff2ba..b703a9d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.physical.config;
import java.util.List;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractMultiple;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
@@ -51,16 +50,6 @@ public class UnionAll extends AbstractMultiple {
}
@Override
- public OperatorCost getCost() {
- OperatorCost cost = new OperatorCost(0,0,0,0);
- for (int i = 0; i < children.length; i++) {
- PhysicalOperator child = children[i];
- cost.add(child.getCost());
- }
- return cost;
- }
-
- @Override
public int getOperatorType() {
return CoreOperatorType.UNION_VALUE;
}
[08/32] git commit: DRILL-735: Refactor Hive storage testcases into a
separate test class
Posted by ja...@apache.org.
DRILL-735: Refactor Hive storage testcases into a separate test class
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c3dfe379
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c3dfe379
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c3dfe379
Branch: refs/heads/master
Commit: c3dfe379e58a9dab5341cbd3a25267eed5ac9bc0
Parents: e31ef07
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jun 13 11:35:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:53:13 2014 -0700
----------------------------------------------------------------------
.../drill/jdbc/test/JdbcTestQueryBase.java | 12 -----
.../apache/drill/jdbc/test/TestHiveStorage.java | 49 ++++++++++++++++++++
.../apache/drill/jdbc/test/TestJdbcQuery.java | 24 ++--------
.../apache/drill/jdbc/test/TestMetadataDDL.java | 9 ++++
.../org/apache/drill/jdbc/test/TestViews.java | 16 +++++++
5 files changed, 78 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3dfe379/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
index 259fa8c..33a25b7 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
@@ -48,18 +48,6 @@ public class JdbcTestQueryBase extends JdbcTest {
}
- @BeforeClass
- public static void generateHive() throws Exception{
- new HiveTestDataGenerator().generateTestData();
-
- // delete tmp workspace directory
- File f = new File("/tmp/drilltest");
- if(f.exists()){
- FileUtils.cleanDirectory(f);
- FileUtils.forceDelete(f);
- }
- }
-
protected void testQuery(String sql) throws Exception{
boolean success = false;
try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3dfe379/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
new file mode 100644
index 0000000..e4ef332
--- /dev/null
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc.test;
+
+import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestHiveStorage extends JdbcTestQueryBase{
+
+ @BeforeClass
+ public static void generateHive() throws Exception{
+ new HiveTestDataGenerator().generateTestData();
+ }
+
+ @Test
+ public void testHiveReadWithDb() throws Exception{
+ testQuery("select * from hive.`default`.kv");
+ testQuery("select key from hive.`default`.kv group by key");
+ testQuery("select * from hive.`default`.allreadsupportedhivedatatypes");
+ }
+
+ @Test
+ public void testHiveWithDate() throws Exception {
+ testQuery("select * from hive.`default`.foodate");
+ testQuery("select date_add(a, time '12:23:33'), b from hive.`default`.foodate");
+ }
+
+ @Test
+ public void testQueryEmptyHiveTable() throws Exception {
+ testQuery("SELECT * FROM hive.`default`.empty_table");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3dfe379/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 44cba3d..e48897f 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.jdbc.test;
-import static org.junit.Assert.assertEquals;
-
+import java.lang.Exception;
+import java.lang.RuntimeException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
@@ -29,28 +29,12 @@ import org.junit.Test;
import com.google.common.base.Function;
+import static org.junit.Assert.assertEquals;
+
public class TestJdbcQuery extends JdbcTestQueryBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
@Test
- public void testHiveReadWithDb() throws Exception{
- testQuery("select * from hive.`default`.kv");
- testQuery("select key from hive.`default`.kv group by key");
- testQuery("select * from hive.`default`.allreadsupportedhivedatatypes");
- }
-
- @Test
- public void testHiveWithDate() throws Exception {
- testQuery("select * from hive.`default`.foodate");
- testQuery("select date_add(a, time '12:23:33'), b from hive.`default`.foodate");
- }
-
- @Test
- public void testQueryEmptyHiveTable() throws Exception {
- testQuery("SELECT * FROM hive.`default`.empty_table");
- }
-
- @Test
@Ignore
public void testJsonQuery() throws Exception{
testQuery("select * from cp.`employee.json`");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3dfe379/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index bb50e40..228bf84 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -18,8 +18,12 @@
package org.apache.drill.jdbc.test;
import com.google.common.base.Function;
+import org.apache.commons.io.FileUtils;
+import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
+import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.File;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
@@ -35,6 +39,11 @@ import static org.junit.Assert.assertTrue;
*/
public class TestMetadataDDL extends JdbcTestQueryBase {
+ @BeforeClass
+ public static void generateHive() throws Exception{
+ new HiveTestDataGenerator().generateTestData();
+ }
+
@Test
public void testInfoSchema() throws Exception{
testQuery("select * from INFORMATION_SCHEMA.SCHEMATA");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3dfe379/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 976c5fa..6f595af 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -18,9 +18,13 @@
package org.apache.drill.jdbc.test;
import com.google.common.base.Function;
+import org.apache.commons.io.FileUtils;
+import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.File;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
@@ -32,6 +36,18 @@ public class TestViews extends JdbcTestQueryBase {
private final static String NEW_LINE = System.getProperty("line.separator");
+ @BeforeClass
+ public static void generateHive() throws Exception{
+ new HiveTestDataGenerator().generateTestData();
+
+ // delete tmp workspace directory
+ File f = new File("/tmp/drilltest");
+ if(f.exists()){
+ FileUtils.cleanDirectory(f);
+ FileUtils.forceDelete(f);
+ }
+ }
+
/** Helper test method for view tests */
private void testViewHelper(final String viewCreate, final String viewName,
final String viewQuery, final String queryResult) throws Exception{
[15/32] git commit: Fix and improve runtime stats profiles - Stop
stats processing while waiting for next. - Fix stats collection in
PartitionSender and ScanBatch - Add stats to all senders - Add wait time to
operator profile.
Posted by ja...@apache.org.
Fix and improve runtime stats profiles
- Stop stats processing while waiting for next.
- Fix stats collection in PartitionSender and ScanBatch
- Add stats to all senders
- Add wait time to operator profile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/fc1a7778
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/fc1a7778
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/fc1a7778
Branch: refs/heads/master
Commit: fc1a7778e2af3b07117f99070530dd5a296ebc6d
Parents: 49a9ff2
Author: Steven Phillips <sp...@maprtech.com>
Authored: Fri Jun 13 13:14:12 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:04:43 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/ops/OperatorStats.java | 20 ++-
.../drill/exec/physical/impl/BaseRootExec.java | 36 +++-
.../physical/impl/RandomReceiverCreator.java | 2 +-
.../drill/exec/physical/impl/ScanBatch.java | 58 +++---
.../drill/exec/physical/impl/ScreenCreator.java | 39 ++--
.../exec/physical/impl/SingleSenderCreator.java | 29 ++-
.../exec/physical/impl/WireRecordBatch.java | 25 ++-
.../BroadcastSenderRootExec.java | 28 ++-
.../impl/mergereceiver/MergingRecordBatch.java | 15 +-
.../PartitionSenderRootExec.java | 18 +-
.../partitionsender/PartitionerTemplate.java | 14 +-
.../drill/exec/record/AbstractRecordBatch.java | 2 +
.../drill/exec/server/rest/ProfileWrapper.java | 177 ++++++++++++-------
.../drill/exec/proto/SchemaUserBitShared.java | 7 +
.../apache/drill/exec/proto/UserBitShared.java | 138 ++++++++++++---
.../drill/exec/proto/beans/OperatorProfile.java | 22 +++
protocol/src/main/protobuf/UserBitShared.proto | 1 +
17 files changed, 466 insertions(+), 165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index 4ac8f74..4afea7b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -41,12 +41,15 @@ public class OperatorStats {
private boolean inProcessing = false;
private boolean inSetup = false;
+ private boolean inWait = false;
protected long processingNanos;
protected long setupNanos;
+ protected long waitNanos;
private long processingMark;
private long setupMark;
+ private long waitMark;
private long schemas;
@@ -89,6 +92,20 @@ public class OperatorStats {
inProcessing = false;
}
+ public void startWait() {
+ assert !inWait;
+ stopProcessing();
+ inWait = true;
+ waitMark = System.nanoTime();
+ }
+
+ public void stopWait() {
+ assert inWait;
+ startProcessing();
+ waitNanos += System.nanoTime() - waitMark;
+ inWait = false;
+ }
+
public void batchReceived(int inputIndex, long records, boolean newSchema) {
recordsReceivedByInput[inputIndex] += records;
batchesReceivedByInput[inputIndex]++;
@@ -103,7 +120,8 @@ public class OperatorStats {
.setOperatorType(operatorType) //
.setOperatorId(operatorId) //
.setSetupNanos(setupNanos) //
- .setProcessNanos(processingNanos);
+ .setProcessNanos(processingNanos)
+ .setWaitNanos(waitNanos);
addAllMetrics(b);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
index 256c106..452052b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
@@ -17,11 +17,25 @@
*/
package org.apache.drill.exec.physical.impl;
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.ops.OperatorStats;
+import org.apache.drill.exec.ops.SenderStats;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatch.IterOutcome;
public abstract class BaseRootExec implements RootExec {
- protected OperatorStats stats = null;
+ protected SenderStats stats = null;
+ protected OperatorContext oContext = null;
+
+ public BaseRootExec(FragmentContext context, PhysicalOperator config) throws OutOfMemoryException {
+ this.stats = new SenderStats(config);
+ context.getStats().addOperatorStats(this.stats);
+ this.oContext = new OperatorContext(config, context, stats);
+ }
@Override
public final boolean next() {
@@ -35,8 +49,24 @@ public abstract class BaseRootExec implements RootExec {
}
}
- public void setStats(OperatorStats stats) {
- this.stats = stats;
+ public final IterOutcome next(RecordBatch b){
+ stats.stopProcessing();
+ IterOutcome next;
+ try {
+ next = b.next();
+ } finally {
+ stats.startProcessing();
+ }
+
+ switch(next){
+ case OK_NEW_SCHEMA:
+ stats.batchReceived(0, b.getRecordCount(), true);
+ break;
+ case OK:
+ stats.batchReceived(0, b.getRecordCount(), false);
+ break;
+ }
+ return next;
}
public abstract boolean innerNext();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
index 966c221..4ff5831 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RandomReceiverCreator.java
@@ -39,7 +39,7 @@ public class RandomReceiverCreator implements BatchCreator<RandomReceiver>{
RawBatchBuffer[] buffers = bufHolder.getBuffers(receiver.getOppositeMajorFragmentId());
assert buffers.length == 1;
RawBatchBuffer buffer = buffers[0];
- return new WireRecordBatch(context, buffer);
+ return new WireRecordBatch(context, buffer, receiver);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index d142ff8..55d3f62 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -127,34 +127,44 @@ public class ScanBatch implements RecordBatch {
@Override
public IterOutcome next() {
- mutator.allocate(MAX_RECORD_CNT);
- while ((recordCount = currentReader.next()) == 0) {
- try {
- if (!readers.hasNext()) {
- currentReader.cleanup();
+ oContext.getStats().startProcessing();
+ try {
+ mutator.allocate(MAX_RECORD_CNT);
+ while ((recordCount = currentReader.next()) == 0) {
+ try {
+ if (!readers.hasNext()) {
+ currentReader.cleanup();
+ releaseAssets();
+ return IterOutcome.NONE;
+ }
+ oContext.getStats().startSetup();
+ try {
+ currentReader.cleanup();
+ currentReader = readers.next();
+ partitionValues = partitionColumns.hasNext() ? partitionColumns.next() : null;
+ currentReader.setup(mutator);
+ mutator.allocate(MAX_RECORD_CNT);
+ addPartitionVectors();
+ } finally {
+ oContext.getStats().stopSetup();
+ }
+ } catch (ExecutionSetupException e) {
+ this.context.fail(e);
releaseAssets();
- return IterOutcome.NONE;
+ return IterOutcome.STOP;
}
- currentReader.cleanup();
- currentReader = readers.next();
- partitionValues = partitionColumns.hasNext() ? partitionColumns.next() : null;
- currentReader.setup(mutator);
- mutator.allocate(MAX_RECORD_CNT);
- addPartitionVectors();
- } catch (ExecutionSetupException e) {
- this.context.fail(e);
- releaseAssets();
- return IterOutcome.STOP;
}
- }
- populatePartitionVectors();
- if (mutator.isNewSchema()) {
- container.buildSchema(SelectionVectorMode.NONE);
- schema = container.getSchema();
- return IterOutcome.OK_NEW_SCHEMA;
- } else {
- return IterOutcome.OK;
+ populatePartitionVectors();
+ if (mutator.isNewSchema()) {
+ container.buildSchema(SelectionVectorMode.NONE);
+ schema = container.getSchema();
+ return IterOutcome.OK_NEW_SCHEMA;
+ } else {
+ return IterOutcome.OK;
+ }
+ } finally {
+ oContext.getStats().stopProcessing();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
index 643552b..86e77d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
@@ -21,7 +21,10 @@ import io.netty.buffer.ByteBuf;
import java.util.List;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.impl.materialize.QueryWritableBatch;
import org.apache.drill.exec.physical.impl.materialize.RecordMaterializer;
@@ -44,14 +47,14 @@ public class ScreenCreator implements RootCreator<Screen>{
@Override
- public RootExec getRoot(FragmentContext context, Screen config, List<RecordBatch> children) {
+ public RootExec getRoot(FragmentContext context, Screen config, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkNotNull(children);
Preconditions.checkArgument(children.size() == 1);
- return new ScreenRoot(context, children.iterator().next());
+ return new ScreenRoot(context, children.iterator().next(), config);
}
- static class ScreenRoot implements RootExec{
+ static class ScreenRoot extends BaseRootExec {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScreenRoot.class);
volatile boolean ok = true;
@@ -62,9 +65,9 @@ public class ScreenCreator implements RootCreator<Screen>{
final UserClientConnection connection;
private RecordMaterializer materializer;
- public ScreenRoot(FragmentContext context, RecordBatch incoming){
+ public ScreenRoot(FragmentContext context, RecordBatch incoming, Screen config) throws OutOfMemoryException {
+ super(context, config);
assert context.getConnection() != null : "A screen root should only be run on the driving node which is connected directly to the client. As such, this should always be true.";
-
this.context = context;
this.incoming = incoming;
this.connection = context.getConnection();
@@ -72,14 +75,14 @@ public class ScreenCreator implements RootCreator<Screen>{
@Override
- public boolean next() {
+ public boolean innerNext() {
if(!ok){
stop();
context.fail(this.listener.ex);
return false;
}
- IterOutcome outcome = incoming.next();
+ IterOutcome outcome = next(incoming);
// logger.debug("Screen Outcome {}", outcome);
switch(outcome){
case STOP: {
@@ -92,7 +95,12 @@ public class ScreenCreator implements RootCreator<Screen>{
.setIsLastChunk(true) //
.build();
QueryWritableBatch batch = new QueryWritableBatch(header);
- connection.sendResult(listener, batch);
+ stats.startWait();
+ try {
+ connection.sendResult(listener, batch);
+ } finally {
+ stats.stopWait();
+ }
sendCount.increment();
return false;
@@ -107,7 +115,12 @@ public class ScreenCreator implements RootCreator<Screen>{
.setIsLastChunk(true) //
.build();
QueryWritableBatch batch = new QueryWritableBatch(header);
- connection.sendResult(listener, batch);
+ stats.startWait();
+ try {
+ connection.sendResult(listener, batch);
+ } finally {
+ stats.stopWait();
+ }
sendCount.increment();
return false;
@@ -119,7 +132,12 @@ public class ScreenCreator implements RootCreator<Screen>{
// context.getStats().batchesCompleted.inc(1);
// context.getStats().recordsCompleted.inc(incoming.getRecordCount());
QueryWritableBatch batch = materializer.convertNext(false);
- connection.sendResult(listener, batch);
+ stats.startWait();
+ try {
+ connection.sendResult(listener, batch);
+ } finally {
+ stats.stopWait();
+ }
sendCount.increment();
return true;
@@ -131,6 +149,7 @@ public class ScreenCreator implements RootCreator<Screen>{
@Override
public void stop() {
sendCount.waitForSendComplete();
+ oContext.close();
incoming.cleanup();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
index 7679701..9e91468 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
@@ -22,7 +22,10 @@ import io.netty.buffer.ByteBuf;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.ops.SenderStats;
import org.apache.drill.exec.physical.config.SingleSender;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
@@ -44,7 +47,7 @@ public class SingleSenderCreator implements RootCreator<SingleSender>{
- private static class SingleSenderRootExec implements RootExec{
+ private static class SingleSenderRootExec extends BaseRootExec {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SingleSenderRootExec.class);
private RecordBatch incoming;
private DataTunnel tunnel;
@@ -53,8 +56,9 @@ public class SingleSenderCreator implements RootCreator<SingleSender>{
private FragmentContext context;
private volatile boolean ok = true;
private final SendingAccountor sendCount = new SendingAccountor();
-
- public SingleSenderRootExec(FragmentContext context, RecordBatch batch, SingleSender config){
+
+ public SingleSenderRootExec(FragmentContext context, RecordBatch batch, SingleSender config) throws OutOfMemoryException {
+ super(context, config);
this.incoming = batch;
assert(incoming != null);
this.handle = context.getHandle();
@@ -65,27 +69,37 @@ public class SingleSenderCreator implements RootCreator<SingleSender>{
}
@Override
- public boolean next() {
+ public boolean innerNext() {
if(!ok){
incoming.kill();
return false;
}
- IterOutcome out = incoming.next();
+ IterOutcome out = next(incoming);
// logger.debug("Outcome of sender next {}", out);
switch(out){
case STOP:
case NONE:
FragmentWritableBatch b2 = FragmentWritableBatch.getEmptyLast(handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), recMajor, 0);
sendCount.increment();
- tunnel.sendRecordBatch(new RecordSendFailure(), b2);
+ stats.startWait();
+ try {
+ tunnel.sendRecordBatch(new RecordSendFailure(), b2);
+ } finally {
+ stats.stopWait();
+ }
return false;
case OK_NEW_SCHEMA:
case OK:
FragmentWritableBatch batch = new FragmentWritableBatch(false, handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), recMajor, 0, incoming.getWritableBatch());
sendCount.increment();
- tunnel.sendRecordBatch(new RecordSendFailure(), batch);
+ stats.startWait();
+ try {
+ tunnel.sendRecordBatch(new RecordSendFailure(), batch);
+ } finally {
+ stats.stopWait();
+ }
return true;
case NOT_YET:
@@ -98,6 +112,7 @@ public class SingleSenderCreator implements RootCreator<SingleSender>{
public void stop() {
ok = false;
sendCount.waitForSendComplete();
+ oContext.close();
incoming.cleanup();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index c7fc813..bc2cdb5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -24,6 +24,9 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OpProfileDef;
+import org.apache.drill.exec.ops.OperatorStats;
+import org.apache.drill.exec.physical.config.RandomReceiver;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.RawFragmentBatch;
@@ -43,14 +46,16 @@ public class WireRecordBatch implements RecordBatch {
private RawFragmentBatchProvider fragProvider;
private FragmentContext context;
private BatchSchema schema;
+ private OperatorStats stats;
- public WireRecordBatch(FragmentContext context, RawFragmentBatchProvider fragProvider) throws OutOfMemoryException {
+ public WireRecordBatch(FragmentContext context, RawFragmentBatchProvider fragProvider, RandomReceiver config) throws OutOfMemoryException {
this.fragProvider = fragProvider;
this.context = context;
// In normal case, batchLoader does not require an allocator. However, in case of splitAndTransfer of a value vector,
// we may need an allocator for the new offset vector. Therefore, here we pass the context's allocator to batchLoader.
this.batchLoader = new RecordBatchLoader(context.getAllocator());
+ this.stats = context.getStats().getOperatorStats(new OpProfileDef(config.getOperatorId(), config.getOperatorType(), 0));
}
@Override
@@ -100,14 +105,22 @@ public class WireRecordBatch implements RecordBatch {
@Override
public IterOutcome next() {
+ stats.startProcessing();
try{
- RawFragmentBatch batch = fragProvider.getNext();
-
- // skip over empty batches. we do this since these are basically control messages.
- while(batch != null && !batch.getHeader().getIsOutOfMemory() && batch.getHeader().getDef().getRecordCount() == 0){
+ RawFragmentBatch batch;
+ try {
+ stats.startWait();
batch = fragProvider.getNext();
+
+ // skip over empty batches. we do this since these are basically control messages.
+ while(batch != null && !batch.getHeader().getIsOutOfMemory() && batch.getHeader().getDef().getRecordCount() == 0){
+ batch = fragProvider.getNext();
+ }
+ } finally {
+ stats.stopWait();
}
+
if (batch == null){
batchLoader.clear();
return IterOutcome.NONE;
@@ -133,6 +146,8 @@ public class WireRecordBatch implements RecordBatch {
}catch(SchemaChangeException | IOException ex){
context.fail(ex);
return IterOutcome.STOP;
+ } finally {
+ stats.stopProcessing();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
index 9c55825..a70cd50 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
@@ -21,8 +21,12 @@ import io.netty.buffer.ByteBuf;
import java.util.List;
+import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.ops.SenderStats;
import org.apache.drill.exec.physical.config.BroadcastSender;
+import org.apache.drill.exec.physical.impl.BaseRootExec;
import org.apache.drill.exec.physical.impl.RootExec;
import org.apache.drill.exec.physical.impl.SendingAccountor;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
@@ -43,7 +47,7 @@ import org.apache.drill.exec.work.ErrorHelper;
* This is useful in cases such as broadcast join where sending the entire table to join
* to all nodes is cheaper than merging and computing all the joins in the same node.
*/
-public class BroadcastSenderRootExec implements RootExec {
+public class BroadcastSenderRootExec extends BaseRootExec {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BroadcastSenderRootExec.class);
private final FragmentContext context;
private final BroadcastSender config;
@@ -54,7 +58,8 @@ public class BroadcastSenderRootExec implements RootExec {
public BroadcastSenderRootExec(FragmentContext context,
RecordBatch incoming,
- BroadcastSender config) {
+ BroadcastSender config) throws OutOfMemoryException {
+ super(context, config);
this.ok = true;
this.context = context;
this.incoming = incoming;
@@ -69,20 +74,25 @@ public class BroadcastSenderRootExec implements RootExec {
}
@Override
- public boolean next() {
+ public boolean innerNext() {
if(!ok) {
context.fail(statusHandler.ex);
return false;
}
- RecordBatch.IterOutcome out = incoming.next();
+ RecordBatch.IterOutcome out = next(incoming);
logger.debug("Outcome of sender next {}", out);
switch(out){
case STOP:
case NONE:
for (int i = 0; i < tunnels.length; ++i) {
FragmentWritableBatch b2 = FragmentWritableBatch.getEmptyLast(handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), config.getOppositeMajorFragmentId(), i);
- tunnels[i].sendRecordBatch(this.statusHandler, b2);
+ stats.startWait();
+ try {
+ tunnels[i].sendRecordBatch(this.statusHandler, b2);
+ } finally {
+ stats.stopWait();
+ }
statusHandler.sendCount.increment();
}
@@ -96,7 +106,12 @@ public class BroadcastSenderRootExec implements RootExec {
}
for (int i = 0; i < tunnels.length; ++i) {
FragmentWritableBatch batch = new FragmentWritableBatch(false, handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), config.getOppositeMajorFragmentId(), i, writableBatch);
- tunnels[i].sendRecordBatch(this.statusHandler, batch);
+ stats.startWait();
+ try {
+ tunnels[i].sendRecordBatch(this.statusHandler, batch);
+ } finally {
+ stats.stopWait();
+ }
statusHandler.sendCount.increment();
}
@@ -135,6 +150,7 @@ public class BroadcastSenderRootExec implements RootExec {
public void stop() {
ok = false;
statusHandler.sendCount.waitForSendComplete();
+ oContext.close();
incoming.cleanup();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 25ee667..a5d80b0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -119,13 +119,16 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
}
private RawFragmentBatch getNext(RawFragmentBatchProvider provider) throws IOException{
- long startNext = System.nanoTime();
- RawFragmentBatch b = provider.getNext();
- if(b != null){
- stats.batchReceived(0, b.getHeader().getDef().getRecordCount(), false);
+ stats.startWait();
+ try {
+ RawFragmentBatch b = provider.getNext();
+ if(b != null){
+ stats.batchReceived(0, b.getHeader().getDef().getRecordCount(), false);
+ }
+ return b;
+ } finally {
+ stats.stopWait();
}
- stats.addLongStat(Metric.NEXT_WAIT_NANOS, System.nanoTime() - startNext);
- return b;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index bb640b4..7535dcc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -56,28 +56,22 @@ public class PartitionSenderRootExec extends BaseRootExec {
private HashPartitionSender operator;
private Partitioner partitioner;
private FragmentContext context;
- private OperatorContext oContext;
private boolean ok = true;
private final SendingAccountor sendCount = new SendingAccountor();
private final int outGoingBatchCount;
private final HashPartitionSender popConfig;
private final StatusHandler statusHandler;
- private final SenderStats stats;
public PartitionSenderRootExec(FragmentContext context,
RecordBatch incoming,
HashPartitionSender operator) throws OutOfMemoryException {
-
+ super(context, operator);
this.incoming = incoming;
this.operator = operator;
this.context = context;
this.outGoingBatchCount = operator.getDestinations().size();
this.popConfig = operator;
this.statusHandler = new StatusHandler(sendCount, context);
- this.stats = new SenderStats(operator);
- context.getStats().addOperatorStats(this.stats);
- setStats(stats);
- this.oContext = new OperatorContext(operator, context, stats);
}
@Override
@@ -90,7 +84,8 @@ public class PartitionSenderRootExec extends BaseRootExec {
return false;
}
- RecordBatch.IterOutcome out = incoming.next();
+ RecordBatch.IterOutcome out = next(incoming);
+
logger.debug("Partitioner.next(): got next record batch with status {}", out);
switch(out){
case NONE:
@@ -122,7 +117,6 @@ public class PartitionSenderRootExec extends BaseRootExec {
partitioner.flushOutgoingBatches(false, true);
partitioner.clear();
}
- // update DeprecatedOutgoingRecordBatch's schema and generate partitioning code
createPartitioner();
} catch (IOException e) {
incoming.kill();
@@ -227,6 +221,12 @@ public class PartitionSenderRootExec extends BaseRootExec {
fieldId,
WritableBatch.getBatchNoHVWrap(0, container, false));
tunnel.sendRecordBatch(statusHandler, writableBatch);
+ stats.startWait();
+ try {
+ tunnel.sendRecordBatch(statusHandler, writableBatch);
+ } finally {
+ stats.stopWait();
+ }
this.sendCount.increment();
fieldId++;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
index 9bb24d4..6a26d30 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
@@ -265,7 +265,12 @@ public abstract class PartitionerTemplate implements Partitioner {
oppositeMinorFragmentId,
getWritableBatch());
- tunnel.sendRecordBatch(statusHandler, writableBatch);
+ stats.startWait();
+ try {
+ tunnel.sendRecordBatch(statusHandler, writableBatch);
+ } finally {
+ stats.stopWait();
+ }
this.sendCount.increment();
} else {
logger.debug("Flush requested on an empty outgoing record batch" + (isLast ? " (last batch)" : ""));
@@ -278,7 +283,12 @@ public abstract class PartitionerTemplate implements Partitioner {
operator.getOppositeMajorFragmentId(),
oppositeMinorFragmentId,
getWritableBatch());
- tunnel.sendRecordBatch(statusHandler, writableBatch);
+ stats.startWait();
+ try {
+ tunnel.sendRecordBatch(statusHandler, writableBatch);
+ } finally {
+ stats.stopWait();
+ }
this.sendCount.increment();
vectorContainer.clear();
return;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index d71b811..72a7d3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -66,7 +66,9 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
}
public final IterOutcome next(int inputIndex, RecordBatch b){
+ stats.stopProcessing();
IterOutcome next = b.next();
+ stats.startProcessing();
switch(next){
case OK_NEW_SCHEMA:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
index a1d4df9..2952c41 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
@@ -23,15 +23,20 @@ import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile;
import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile;
import org.apache.drill.exec.proto.UserBitShared.OperatorProfile;
import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.proto.UserBitShared.StreamProfile;
+import java.text.DateFormat;
import java.text.NumberFormat;
+import java.text.SimpleDateFormat;
import java.util.Collections;
+import java.util.Date;
import java.util.List;
import java.util.Locale;
public class ProfileWrapper {
NumberFormat format = NumberFormat.getInstance(Locale.US);
+ DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
public QueryProfile profile;
@@ -46,10 +51,14 @@ public class ProfileWrapper {
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
- builder.append("MAJOR FRAGMENTS\nid\tmin\tavg\tmax\t(time in ms)\n\n" + listMajorFragments());
+ builder.append("MAJOR FRAGMENTS\nid\tfirst start\tlast start\tfirst end\tlast end\tmin\tavg\tmax\t(time in ms)\n\n" + listMajorFragments());
builder.append("\n");
for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) {
- builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), new MajorFragmentWrapper(majorProfile).toString()));
+ builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), printOperatorsInMajor(majorProfile)));
+ }
+ builder.append("\n");
+ for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) {
+ builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), printMinorFragmentsInMajor(majorProfile)));
}
return builder.toString();
}
@@ -58,8 +67,12 @@ public class ProfileWrapper {
StringBuilder builder = new StringBuilder();
for (MajorFragmentProfile m : profile.getFragmentProfileList()) {
List<Long> totalTimes = Lists.newArrayList();
+ List<Long> startTimes = Lists.newArrayList();
+ List<Long> endTimes = Lists.newArrayList();
for (MinorFragmentProfile minorFragmentProfile : m.getMinorFragmentProfileList()) {
totalTimes.add(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime());
+ startTimes.add(minorFragmentProfile.getStartTime());
+ endTimes.add(minorFragmentProfile.getEndTime());
}
long min = Collections.min(totalTimes);
long max = Collections.max(totalTimes);
@@ -67,82 +80,120 @@ public class ProfileWrapper {
for (Long l : totalTimes) {
sum += l;
}
+ long firstStart = Collections.min(startTimes);
+ long lastStart = Collections.max(startTimes);
+ long firstEnd = Collections.min(endTimes);
+ long lastEnd = Collections.max(endTimes);
long avg = sum / totalTimes.size();
- builder.append(String.format("%d\t%s\t%s\t%s\n", m.getMajorFragmentId(), format.format(min), format.format(avg), format.format(max)));
+ builder.append(String.format("%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", m.getMajorFragmentId(), dateFormat.format(new Date(firstStart)),
+ dateFormat.format(new Date(lastStart)), dateFormat.format(new Date(firstEnd)), dateFormat.format(new Date(lastEnd)),
+ format.format(min), format.format(avg), format.format(max)));
}
return builder.toString();
}
- public class MajorFragmentWrapper {
- MajorFragmentProfile majorFragmentProfile;
+ public String printMinorFragmentsInMajor(MajorFragmentProfile majorFragmentProfile) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("id\tstart\tend\ttotal time (ms)\tmax records\tbatches\n");
+ for (MinorFragmentProfile m : majorFragmentProfile.getMinorFragmentProfileList()) {
+ long startTime = m.getStartTime();
+ long endTime = m.getEndTime();
+
+ List<OperatorProfile> operators = m.getOperatorProfileList();
+ OperatorProfile biggest = null;
+ int biggestIncomingRecords = 0;
+ for (OperatorProfile oProfile : operators) {
+ if (biggest == null) {
+ biggest = oProfile;
+ int incomingRecordCount = 0;
+ for (StreamProfile streamProfile : oProfile.getInputProfileList()) {
+ incomingRecordCount += streamProfile.getRecords();
+ }
+ biggestIncomingRecords = incomingRecordCount;
+ } else {
+ int incomingRecordCount = 0;
+ for (StreamProfile streamProfile : oProfile.getInputProfileList()) {
+ incomingRecordCount += streamProfile.getRecords();
+ }
+ if (incomingRecordCount > biggestIncomingRecords) {
+ biggest = oProfile;
+ biggestIncomingRecords = incomingRecordCount;
+ }
+ }
+ }
- public MajorFragmentWrapper(MajorFragmentProfile majorFragmentProfile) {
- this.majorFragmentProfile = majorFragmentProfile;
- }
+ int biggestBatches = 0;
+ for (StreamProfile sProfile : biggest.getInputProfileList()) {
+ biggestBatches += sProfile.getBatches();
+ }
- @Override
- public String toString() {
- return String.format("Minor Fragments\nid\ttotal time (ms)\n%s\nOperators\nid\ttype\tmin\tavg\tmax\t(time in ns)\n%s\n", new MinorFragmentsInMajor().toString(), new OperatorsInMajor().toString());
+ builder.append(String.format("%d\t%s\t%s\t%s\t%s\t%s\n", m.getMinorFragmentId(), dateFormat.format(new Date(startTime)),
+ dateFormat.format(new Date(endTime)), format.format(endTime - startTime), biggestIncomingRecords, biggestBatches));
}
+ return builder.toString();
+ }
- public class MinorFragmentsInMajor {
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- for (MinorFragmentProfile minorFragmentProfile: majorFragmentProfile.getMinorFragmentProfileList()) {
- builder.append(String.format("%d\t%s\n", minorFragmentProfile.getMinorFragmentId(), format.format(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime())));
+ public String printOperatorsInMajor(MajorFragmentProfile majorFragmentProfile) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("id\ttype\tp min\tp avg\tp max\ts min\ts avg\ts max\tw min\tw avg\tw max\n");
+ int numOperators = majorFragmentProfile.getMinorFragmentProfile(0).getOperatorProfileCount();
+ int numFragments = majorFragmentProfile.getMinorFragmentProfileCount();
+ long[][] processing = new long[numOperators + 1][numFragments];
+ long[][] setup = new long[numOperators + 1][numFragments];
+ long[][] wait = new long[numOperators + 1][numFragments];
+ CoreOperatorType[] operatorTypes = new CoreOperatorType[numOperators + 1];
+
+ for (int i = 0; i < numFragments; i++) {
+ MinorFragmentProfile minorProfile = majorFragmentProfile.getMinorFragmentProfile(i);
+ for (int j = 0; j < numOperators; j++) {
+ OperatorProfile operatorProfile = minorProfile.getOperatorProfile(j);
+ int operatorId = operatorProfile.getOperatorId();
+ processing[operatorId][i] = operatorProfile.getProcessNanos();
+ setup[operatorId][i] = operatorProfile.getSetupNanos();
+ wait[operatorId][i] = operatorProfile.getWaitNanos();
+ if (i == 0) {
+ operatorTypes[operatorId] = CoreOperatorType.valueOf(operatorProfile.getOperatorType());
}
- return builder.toString();
}
}
- public class OperatorsInMajor {
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- int numOperators = majorFragmentProfile.getMinorFragmentProfile(0).getOperatorProfileCount();
- int numFragments = majorFragmentProfile.getMinorFragmentProfileCount();
- long[][] values = new long[numOperators + 1][numFragments];
- CoreOperatorType[] operatorTypes = new CoreOperatorType[numOperators + 1];
-
- for (int i = 0; i < numFragments; i++) {
- MinorFragmentProfile minorProfile = majorFragmentProfile.getMinorFragmentProfile(i);
- for (int j = 0; j < numOperators; j++) {
- OperatorProfile operatorProfile = minorProfile.getOperatorProfile(j);
- int operatorId = operatorProfile.getOperatorId();
- values[operatorId][i] = operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos();
- if (i == 0) {
- operatorTypes[operatorId] = CoreOperatorType.valueOf(operatorProfile.getOperatorType());
- }
- }
- }
-
- for (int j = 0; j < numOperators + 1; j++) {
- if (operatorTypes[j] == null) {
- continue;
- }
- long min = Long.MAX_VALUE;
- long max = Long.MIN_VALUE;
- long sum = 0;
-
- for (int i = 0; i < numFragments; i++) {
- min = Math.min(min, values[j][i]);
- max = Math.max(max, values[j][i]);
- sum += values[j][i];
- }
+ for (int j = 0; j < numOperators + 1; j++) {
+ if (operatorTypes[j] == null) {
+ continue;
+ }
+ long processingMin = Long.MAX_VALUE;
+ long processingMax = Long.MIN_VALUE;
+ long processingSum = 0;
+ long setupMin = Long.MAX_VALUE;
+ long setupMax = Long.MIN_VALUE;
+ long setupSum = 0;
+ long waitMin = Long.MAX_VALUE;
+ long waitMax = Long.MIN_VALUE;
+ long waitSum = 0;
+
+ for (int i = 0; i < numFragments; i++) {
+ processingMin = Math.min(processingMin, processing[j][i]);
+ processingMax = Math.max(processingMax, processing[j][i]);
+ processingSum += processing[j][i];
+
+ setupMin = Math.min(setupMin, setup[j][i]);
+ setupMax = Math.max(setupMax, setup[j][i]);
+ setupSum += setup[j][i];
+
+ waitMin = Math.min(waitMin, wait[j][i]);
+ waitMax = Math.max(waitMax, wait[j][i]);
+ waitSum += wait[j][i];
+ }
- long avg = sum / numFragments;
+ long processingAvg = processingSum / numFragments;
+ long setupAvg = setupSum / numFragments;
+ long waitAvg = waitSum / numFragments;
- builder.append(String.format("%d\t%s\t%s\t%s\t%s\n", j, operatorTypes[j].toString(), format.format(min), format.format(avg), format.format(max)));
- }
- return builder.toString();
- }
+ builder.append(String.format("%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j, operatorTypes[j].toString(),
+ format.format(processingMin/1000/1000), format.format(processingAvg/1000/1000), format.format(processingMax/1000/1000),
+ format.format(setupMin/1000/1000), format.format(setupAvg/1000/1000), format.format(setupMax/1000/1000),
+ format.format(waitMin/1000/1000), format.format(waitAvg/1000/1000), format.format(waitMax/1000/1000)));
}
+ return builder.toString();
}
-
-
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
index b4718bb..23fcf21 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java
@@ -1755,6 +1755,8 @@ public final class SchemaUserBitShared
for(org.apache.drill.exec.proto.UserBitShared.MetricValue metric : message.getMetricList())
output.writeObject(8, metric, org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true);
+ if(message.hasWaitNanos())
+ output.writeInt64(9, message.getWaitNanos(), false);
}
public boolean isInitialized(org.apache.drill.exec.proto.UserBitShared.OperatorProfile message)
{
@@ -1817,6 +1819,9 @@ public final class SchemaUserBitShared
builder.addMetric(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.MetricValue.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.MERGE));
break;
+ case 9:
+ builder.setWaitNanos(input.readInt64());
+ break;
default:
input.handleUnknownField(number, this);
}
@@ -1864,6 +1869,7 @@ public final class SchemaUserBitShared
case 6: return "processNanos";
case 7: return "localMemoryAllocated";
case 8: return "metric";
+ case 9: return "waitNanos";
default: return null;
}
}
@@ -1882,6 +1888,7 @@ public final class SchemaUserBitShared
fieldMap.put("processNanos", 6);
fieldMap.put("localMemoryAllocated", 7);
fieldMap.put("metric", 8);
+ fieldMap.put("waitNanos", 9);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index c100968..faeba6f 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -13836,6 +13836,16 @@ public final class UserBitShared {
*/
org.apache.drill.exec.proto.UserBitShared.MetricValueOrBuilder getMetricOrBuilder(
int index);
+
+ // optional int64 wait_nanos = 9;
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ boolean hasWaitNanos();
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ long getWaitNanos();
}
/**
* Protobuf type {@code exec.shared.OperatorProfile}
@@ -13929,6 +13939,11 @@ public final class UserBitShared {
metric_.add(input.readMessage(org.apache.drill.exec.proto.UserBitShared.MetricValue.PARSER, extensionRegistry));
break;
}
+ case 72: {
+ bitField0_ |= 0x00000020;
+ waitNanos_ = input.readInt64();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -14127,6 +14142,22 @@ public final class UserBitShared {
return metric_.get(index);
}
+ // optional int64 wait_nanos = 9;
+ public static final int WAIT_NANOS_FIELD_NUMBER = 9;
+ private long waitNanos_;
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ public boolean hasWaitNanos() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ public long getWaitNanos() {
+ return waitNanos_;
+ }
+
private void initFields() {
inputProfile_ = java.util.Collections.emptyList();
operatorId_ = 0;
@@ -14135,6 +14166,7 @@ public final class UserBitShared {
processNanos_ = 0L;
localMemoryAllocated_ = 0L;
metric_ = java.util.Collections.emptyList();
+ waitNanos_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -14169,6 +14201,9 @@ public final class UserBitShared {
for (int i = 0; i < metric_.size(); i++) {
output.writeMessage(8, metric_.get(i));
}
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeInt64(9, waitNanos_);
+ }
getUnknownFields().writeTo(output);
}
@@ -14206,6 +14241,10 @@ public final class UserBitShared {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(8, metric_.get(i));
}
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(9, waitNanos_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -14346,6 +14385,8 @@ public final class UserBitShared {
} else {
metricBuilder_.clear();
}
+ waitNanos_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000080);
return this;
}
@@ -14412,6 +14453,10 @@ public final class UserBitShared {
} else {
result.metric_ = metricBuilder_.build();
}
+ if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.waitNanos_ = waitNanos_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -14495,6 +14540,9 @@ public final class UserBitShared {
}
}
}
+ if (other.hasWaitNanos()) {
+ setWaitNanos(other.getWaitNanos());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -15167,6 +15215,39 @@ public final class UserBitShared {
return metricBuilder_;
}
+ // optional int64 wait_nanos = 9;
+ private long waitNanos_ ;
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ public boolean hasWaitNanos() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ public long getWaitNanos() {
+ return waitNanos_;
+ }
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ public Builder setWaitNanos(long value) {
+ bitField0_ |= 0x00000080;
+ waitNanos_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int64 wait_nanos = 9;</code>
+ */
+ public Builder clearWaitNanos() {
+ bitField0_ = (bitField0_ & ~0x00000080);
+ waitNanos_ = 0L;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:exec.shared.OperatorProfile)
}
@@ -16434,38 +16515,39 @@ public final class UserBitShared {
"le\022\022\n\nstart_time\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003" +
"\022\023\n\013memory_used\030\007 \001(\003\022\027\n\017max_memory_used" +
"\030\010 \001(\003\022(\n\010endpoint\030\t \001(\0132\026.exec.Drillbit" +
- "Endpoint\"\346\001\n\017OperatorProfile\0221\n\rinput_pr" +
+ "Endpoint\"\372\001\n\017OperatorProfile\0221\n\rinput_pr" +
"ofile\030\001 \003(\0132\032.exec.shared.StreamProfile\022" +
"\023\n\013operator_id\030\003 \001(\005\022\025\n\roperator_type\030\004 " +
"\001(\005\022\023\n\013setup_nanos\030\005 \001(\003\022\025\n\rprocess_nano",
"s\030\006 \001(\003\022\036\n\026local_memory_allocated\030\007 \001(\003\022" +
"(\n\006metric\030\010 \003(\0132\030.exec.shared.MetricValu" +
- "e\"B\n\rStreamProfile\022\017\n\007records\030\001 \001(\003\022\017\n\007b" +
- "atches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013MetricV" +
- "alue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong_value\030\002 " +
- "\001(\003\022\024\n\014double_value\030\003 \001(\001*5\n\nRpcChannel\022" +
- "\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004USER\020\002" +
- "*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010P" +
- "HYSICAL\020\003*k\n\rFragmentState\022\013\n\007SENDING\020\000\022" +
- "\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n",
- "\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005*\362" +
- "\004\n\020CoreOperatorType\022\021\n\rSINGLE_SENDER\020\000\022\024" +
- "\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH" +
- "_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOI" +
- "N\020\005\022\031\n\025HASH_PARTITION_SENDER\020\006\022\t\n\005LIMIT\020" +
- "\007\022\024\n\020MERGING_RECEIVER\020\010\022\034\n\030ORDERED_PARTI" +
- "TION_SENDER\020\t\022\013\n\007PROJECT\020\n\022\023\n\017RANDOM_REC" +
- "EIVER\020\013\022\020\n\014RANGE_SENDER\020\014\022\n\n\006SCREEN\020\r\022\034\n" +
- "\030SELECTION_VECTOR_REMOVER\020\016\022\027\n\023STREAMING" +
- "_AGGREGATE\020\017\022\016\n\nTOP_N_SORT\020\020\022\021\n\rEXTERNAL",
- "_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNION\020\023\022\014\n\010OLD_SOR" +
- "T\020\024\022\032\n\026PARQUET_ROW_GROUP_SCAN\020\025\022\021\n\rHIVE_" +
- "SUB_SCAN\020\026\022\025\n\021SYSTEM_TABLE_SCAN\020\027\022\021\n\rMOC" +
- "K_SUB_SCAN\020\030\022\022\n\016PARQUET_WRITER\020\031\022\023\n\017DIRE" +
- "CT_SUB_SCAN\020\032\022\017\n\013TEXT_WRITER\020\033\022\021\n\rTEXT_S" +
- "UB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN\020\035\022\030\n\024INFO_SCH" +
- "EMA_SUB_SCAN\020\036\022\013\n\007FLATTEN\020\037B.\n\033org.apach" +
- "e.drill.exec.protoB\rUserBitSharedH\001"
+ "e\022\022\n\nwait_nanos\030\t \001(\003\"B\n\rStreamProfile\022\017" +
+ "\n\007records\030\001 \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007sche" +
+ "mas\030\003 \001(\003\"J\n\013MetricValue\022\021\n\tmetric_id\030\001 " +
+ "\001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014double_value\030" +
+ "\003 \001(\001*5\n\nRpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010" +
+ "BIT_DATA\020\001\022\010\n\004USER\020\002*/\n\tQueryType\022\007\n\003SQL" +
+ "\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmen" +
+ "tState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATI",
+ "ON\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCE" +
+ "LLED\020\004\022\n\n\006FAILED\020\005*\362\004\n\020CoreOperatorType\022" +
+ "\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001" +
+ "\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH" +
+ "_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITIO" +
+ "N_SENDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVE" +
+ "R\020\010\022\034\n\030ORDERED_PARTITION_SENDER\020\t\022\013\n\007PRO" +
+ "JECT\020\n\022\023\n\017RANDOM_RECEIVER\020\013\022\020\n\014RANGE_SEN" +
+ "DER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_RE" +
+ "MOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_",
+ "N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t" +
+ "\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_G" +
+ "ROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM" +
+ "_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQ" +
+ "UET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEX" +
+ "T_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SU" +
+ "B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\013\n\007FL" +
+ "ATTEN\020\037B.\n\033org.apache.drill.exec.protoB\r" +
+ "UserBitSharedH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -16549,7 +16631,7 @@ public final class UserBitShared {
internal_static_exec_shared_OperatorProfile_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_exec_shared_OperatorProfile_descriptor,
- new java.lang.String[] { "InputProfile", "OperatorId", "OperatorType", "SetupNanos", "ProcessNanos", "LocalMemoryAllocated", "Metric", });
+ new java.lang.String[] { "InputProfile", "OperatorId", "OperatorType", "SetupNanos", "ProcessNanos", "LocalMemoryAllocated", "Metric", "WaitNanos", });
internal_static_exec_shared_StreamProfile_descriptor =
getDescriptor().getMessageTypes().get(13);
internal_static_exec_shared_StreamProfile_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
index b9ed2e4..f1b1acc 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/OperatorProfile.java
@@ -56,6 +56,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
private long processNanos;
private long localMemoryAllocated;
private List<MetricValue> metric;
+ private long waitNanos;
public OperatorProfile()
{
@@ -155,6 +156,19 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
return this;
}
+ // waitNanos
+
+ public long getWaitNanos()
+ {
+ return waitNanos;
+ }
+
+ public OperatorProfile setWaitNanos(long waitNanos)
+ {
+ this.waitNanos = waitNanos;
+ return this;
+ }
+
// java serialization
public void readExternal(ObjectInput in) throws IOException
@@ -236,6 +250,9 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
message.metric.add(input.mergeObject(null, MetricValue.getSchema()));
break;
+ case 9:
+ message.waitNanos = input.readInt64();
+ break;
default:
input.handleUnknownField(number, this);
}
@@ -279,6 +296,9 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
}
}
+
+ if(message.waitNanos != 0)
+ output.writeInt64(9, message.waitNanos, false);
}
public String getFieldName(int number)
@@ -292,6 +312,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
case 6: return "processNanos";
case 7: return "localMemoryAllocated";
case 8: return "metric";
+ case 9: return "waitNanos";
default: return null;
}
}
@@ -312,6 +333,7 @@ public final class OperatorProfile implements Externalizable, Message<OperatorPr
__fieldMap.put("processNanos", 6);
__fieldMap.put("localMemoryAllocated", 7);
__fieldMap.put("metric", 8);
+ __fieldMap.put("waitNanos", 9);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc1a7778/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index b754ee5..eb56efb 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -142,6 +142,7 @@ message OperatorProfile {
optional int64 process_nanos = 6;
optional int64 local_memory_allocated = 7;
repeated MetricValue metric = 8;
+ optional int64 wait_nanos = 9;
}
message StreamProfile {
[32/32] git commit: Merge fixes.
Posted by ja...@apache.org.
Merge fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f86639c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f86639c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f86639c3
Branch: refs/heads/master
Commit: f86639c31a61592eb320f7ebc833c84eff199c43
Parents: 38dde9e
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Jun 16 09:45:51 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 13:51:36 2014 -0700
----------------------------------------------------------------------
.../drill/common/logical/data/Flatten.java | 24 ++++++------
.../drill/exec/cache/infinispan/ICache.java | 6 ++-
.../drill/exec/cache/local/LocalCache.java | 8 ++--
.../drill/exec/physical/config/Flatten.java | 12 ------
.../exec/physical/impl/WireRecordBatch.java | 3 +-
.../PartitionSenderRootExec.java | 1 -
.../exec/planner/fragment/PlanningSet.java | 41 +++++++++++++-------
.../exec/planner/physical/FlattenPrel.java | 3 +-
.../exec/planner/sql/parser/SqlCreateTable.java | 5 ++-
.../exec/planner/sql/parser/SqlCreateView.java | 7 ++--
.../planner/sql/parser/SqlDescribeTable.java | 4 +-
.../exec/planner/sql/parser/SqlDropView.java | 3 ++
.../exec/planner/sql/parser/SqlShowFiles.java | 5 ++-
.../exec/planner/sql/parser/SqlShowSchemas.java | 2 +
.../exec/planner/sql/parser/SqlShowTables.java | 2 +
.../exec/store/hive/HiveFieldConverter.java | 5 ++-
.../apache/drill/exec/work/foreman/Foreman.java | 10 +++--
.../exec/cache/TestCacheSerialization.java | 10 ++---
.../exec/physical/impl/SimpleRootExec.java | 5 ++-
.../exec/physical/impl/TestCastFunctions.java | 14 +++++++
20 files changed, 102 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java b/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
index dce509f..07485d6 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
@@ -29,13 +29,14 @@ import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
import java.util.Iterator;
@JsonTypeName("flatten")
-public class Flatten extends SingleInputOperator{
+public class Flatten extends SingleInputOperator {
private final FieldReference name;
private final LogicalExpression expr;
private final boolean drop;
@JsonCreator
- public Flatten(@JsonProperty("name") FieldReference name, @JsonProperty("expr") LogicalExpression expr, @JsonProperty("drop") boolean drop) {
+ public Flatten(@JsonProperty("name") FieldReference name, @JsonProperty("expr") LogicalExpression expr,
+ @JsonProperty("drop") boolean drop) {
this.name = name;
this.expr = expr;
this.drop = drop;
@@ -53,17 +54,14 @@ public class Flatten extends SingleInputOperator{
return drop;
}
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitFlatten(this, value);
- }
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitFlatten(this, value);
+ }
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
index d3b63db..6627a89 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
@@ -23,9 +23,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
-import com.google.common.collect.Maps;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.Counter;
@@ -55,6 +57,8 @@ import org.jgroups.protocols.COUNTER;
import org.jgroups.protocols.FRAG2;
import org.jgroups.stack.ProtocolStack;
+import com.google.common.collect.Maps;
+
public class ICache implements DistributedCache{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ICache.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
index 31ab909..7328257 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/local/LocalCache.java
@@ -29,7 +29,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.common.config.DrillConfig;
@@ -42,8 +46,6 @@ import org.apache.drill.exec.cache.DrillSerializable;
import org.apache.drill.exec.exception.DrillbitStartupException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
-import org.apache.drill.exec.proto.BitControl.PlanFragment;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
index 2bcaab3..d123d2b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
@@ -17,11 +17,9 @@
*/
package org.apache.drill.exec.physical.config;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -44,16 +42,6 @@ public class Flatten extends AbstractSingle {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(0, 0, 1000, child.getSize().getRecordCount());
- }
-
- @Override
- public Size getSize() {
- return child.getSize();
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new Flatten(child);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index bc2cdb5..2a19ba7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -34,6 +34,7 @@ import org.apache.drill.exec.record.RawFragmentBatchProvider;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
@@ -166,5 +167,5 @@ public class WireRecordBatch implements RecordBatch {
public VectorContainer getOutgoingContainer() {
throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index 7535dcc..23296fb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -220,7 +220,6 @@ public class PartitionSenderRootExec extends BaseRootExec {
operator.getOppositeMajorFragmentId(),
fieldId,
WritableBatch.getBatchNoHVWrap(0, container, false));
- tunnel.sendRecordBatch(statusHandler, writableBatch);
stats.startWait();
try {
tunnel.sendRecordBatch(statusHandler, writableBatch);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
index 3164228..da4380f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
@@ -25,28 +25,43 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.google.common.collect.Maps;
-public class PlanningSet implements Iterable<Wrapper>{
+public class PlanningSet implements Iterable<Wrapper> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PlanningSet.class);
-
+
private Map<Fragment, Wrapper> fragmentMap = Maps.newHashMap();
+ private int majorFragmentIdIndex = 0;
- PlanningSet(){
+ PlanningSet() {
}
- public void addAffinity(Fragment n, DrillbitEndpoint endpoint, float affinity){
+ public void addAffinity(Fragment n, DrillbitEndpoint endpoint, float affinity) {
get(n).addEndpointAffinity(endpoint, affinity);
}
-
- public void setWidth(Fragment n, int width){
+
+ public void setWidth(Fragment n, int width) {
get(n).setWidth(width);
}
-
- Wrapper get(Fragment node){
+
+ Wrapper get(Fragment node) {
Wrapper wrapper = fragmentMap.get(node);
- if(wrapper == null){
- int majorFragmentId = node.getRoot().getOperatorId() == 0 ? 0 : node.getSendingExchange().getChild().getOperatorId() >> 16;
+ if (wrapper == null) {
+
+ int majorFragmentId = 0;
+
+ // If there is a sending exchange, we need to number other than zero.
+ if (node.getSendingExchange() != null) {
+
+ // assign the upper 16 bits as the major fragment id.
+ majorFragmentId = node.getSendingExchange().getChild().getOperatorId() >> 16;
+
+ // if they are not assigned, that means we mostly likely have an externally generated plan. in this case, come up with a major fragmentid.
+ if (majorFragmentId == 0) {
+ majorFragmentId = majorFragmentIdIndex;
+ }
+ }
wrapper = new Wrapper(node, majorFragmentId);
- fragmentMap.put(node, wrapper);
+ fragmentMap.put(node, wrapper);
+ majorFragmentIdIndex++;
}
return wrapper;
}
@@ -60,7 +75,5 @@ public class PlanningSet implements Iterable<Wrapper>{
public String toString() {
return "FragmentPlanningSet:\n" + fragmentMap.values() + "]";
}
-
-
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
index f541422..3c19bae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
@@ -35,8 +35,7 @@ public class FlattenPrel extends SingleRel implements Prel {
@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
Flatten p = new Flatten(((Prel) getChild()).getPhysicalOperator(creator));
- p.setOperatorId(creator.getOperatorId(this));
- return p;
+ return creator.addMetadata(this, p);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
index 0f0cf0a..0170735 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
@@ -17,16 +17,17 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.CreateTableHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlNodeList;
import org.eigenbase.sql.SqlOperator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
index 6f798b8..3666b4c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
@@ -17,14 +17,14 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ViewHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.SqlLiteral;
@@ -35,7 +35,8 @@ import org.eigenbase.sql.SqlSpecialOperator;
import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
public class SqlCreateView extends DrillSqlCall {
public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_VIEW", SqlKind.OTHER){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
index 8b022b0..15da822 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
@@ -17,15 +17,17 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import com.google.common.collect.Lists;
+import java.util.List;
import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.DescribeTableHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.SqlSpecialOperator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
index b86bd20..1d94390 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.sql.parser;
+import java.util.Collections;
import java.util.List;
import net.hydromatic.optiq.tools.Planner;
@@ -24,8 +25,10 @@ import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ViewHandler.DropView;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.SqlSpecialOperator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
index c929db6..690e5fd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.sql.parser;
+import java.util.Collections;
import java.util.List;
import net.hydromatic.optiq.tools.Planner;
@@ -24,16 +25,16 @@ import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ShowFileHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.SqlSpecialOperator;
import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import com.google.common.collect.Lists;
-
/**
* Sql parse tree node to represent statement:
* SHOW FILES [{FROM | IN} db_name] [LIKE 'pattern' | WHERE expr]
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
index 03e4e1d..faa69d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
@@ -24,7 +24,9 @@ import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ShowSchemasHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.SqlSpecialOperator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
index ae95944..e66199a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
@@ -24,8 +24,10 @@ import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ShowTablesHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.SqlSpecialOperator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
index 4ce5e98..5095d90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.store.hive;
-import com.beust.jcommander.internal.Maps;
+import java.util.Map;
+
import org.apache.drill.exec.vector.NullableBigIntVector;
import org.apache.drill.exec.vector.NullableBitVector;
import org.apache.drill.exec.vector.NullableDateVector;
@@ -52,7 +53,7 @@ import org.apache.hadoop.io.Text;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
-import java.util.Map;
+import com.google.common.collect.Maps;
public abstract class HiveFieldConverter {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 1629f8c..826a22f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -339,10 +339,12 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
for(PhysicalOperator ops : plan.getSortedOperators()){
size += ops.getCost();
}
- if(queuingEnabled && size > this.queueThreshold){
- this.lease = largeSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
- }else{
- this.lease = smallSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
+ if(queuingEnabled){
+ if(size > this.queueThreshold){
+ this.lease = largeSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
+ }else{
+ this.lease = smallSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
+ }
}
QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getSessionOptionList(), context.getCurrentEndpoint(),
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestCacheSerialization.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestCacheSerialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestCacheSerialization.java
index 16680b1..d548f6a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestCacheSerialization.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestCacheSerialization.java
@@ -60,10 +60,10 @@ public class TestCacheSerialization extends ExecTest {
private static final DrillConfig CONFIG = DrillConfig.create();
@Test
- public void protobufSerialization() {
+ public void protobufSerialization() throws Exception {
DistributedMap<String, FragmentHandle> map = ICACHE.getMap(CacheConfig.newBuilder(FragmentHandle.class).proto().build());
FragmentHandle s = FragmentHandle.newBuilder().setMajorFragmentId(1).setMinorFragmentId(1).setQueryId(QueryId.newBuilder().setPart1(74).setPart2(66).build()).build();
- map.put("1", s);
+ map.put("1", s).get();
for(int i =0; i < 2; i++){
FragmentHandle s2 = map.get("1");
Assert.assertEquals(s, s2);
@@ -71,10 +71,10 @@ public class TestCacheSerialization extends ExecTest {
}
@Test
- public void jacksonSerialization(){
+ public void jacksonSerialization() throws Exception {
OptionValue v = OptionValue.createBoolean(OptionType.SESSION, "my test option", true);
DistributedMap<String, OptionValue> map = ICACHE.getMap(CacheConfig.newBuilder(OptionValue.class).jackson().build());
- map.put("1", v);
+ map.put("1", v).get();
for(int i = 0; i < 5; i++){
OptionValue v2 = map.get("1");
Assert.assertEquals(v, v2);
@@ -114,7 +114,7 @@ public class TestCacheSerialization extends ExecTest {
CachedVectorContainer wrap = new CachedVectorContainer(batch, ALLOCATOR);
DistributedMultiMap<String, CachedVectorContainer> mmap = ICACHE.getMultiMap(OrderedPartitionRecordBatch.MULTI_CACHE_CONFIG);
- mmap.put("vectors", wrap);
+ mmap.put("vectors", wrap).get();
for(int x =0; x < 2; x++){
CachedVectorContainer newWrap = (CachedVectorContainer) mmap.get("vectors").iterator().next();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
index 7d04340..7dce6e0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
@@ -38,10 +38,11 @@ public class SimpleRootExec implements RootExec, Iterable<ValueVector>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleRootExec.class);
private RecordBatch incoming;
-
+ private ScreenRoot screenRoot;
public SimpleRootExec(RootExec e){
if(e instanceof ScreenRoot){
incoming = ((ScreenRoot)e).getIncoming();
+ screenRoot = (ScreenRoot) e;
}else{
throw new UnsupportedOperationException();
}
@@ -80,7 +81,7 @@ public class SimpleRootExec implements RootExec, Iterable<ValueVector>{
@Override
public void stop() {
- incoming.cleanup();
+ screenRoot.stop();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f86639c3/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 702edb6..73b7b53 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -110,6 +110,8 @@ public class TestCastFunctions extends PopUnitTestBase{
assertEquals(5, count);
}
+ exec.stop();
+
context.close();
allocator.close();
@@ -155,6 +157,8 @@ public class TestCastFunctions extends PopUnitTestBase{
assertEquals(5, count);
}
+ exec.stop();
+
context.close();
allocator.close();
@@ -198,6 +202,8 @@ public class TestCastFunctions extends PopUnitTestBase{
assertEquals(5, count);
}
+ exec.stop();
+
context.close();
allocator.close();
@@ -242,6 +248,8 @@ public class TestCastFunctions extends PopUnitTestBase{
assertEquals(5, count);
}
+ exec.stop();
+
context.close();
allocator.close();
@@ -286,6 +294,8 @@ public class TestCastFunctions extends PopUnitTestBase{
assertEquals(5, count);
}
+ exec.stop();
+
context.close();
allocator.close();
@@ -330,6 +340,7 @@ public class TestCastFunctions extends PopUnitTestBase{
}
assertEquals(5, count);
}
+ exec.stop();
context.close();
allocator.close();
@@ -374,6 +385,7 @@ public class TestCastFunctions extends PopUnitTestBase{
}
assertEquals(5, count);
}
+ exec.stop();
context.close();
allocator.close();
@@ -408,6 +420,8 @@ public class TestCastFunctions extends PopUnitTestBase{
while(exec.next()){
}
+ exec.stop();
+
context.close();
allocator.close();
[05/32] git commit: DRILL-735: Refactor HiveRecordReader
Posted by ja...@apache.org.
DRILL-735: Refactor HiveRecordReader
+ Create HiveFieldConverter classes for each Hive data type to extract value from Hive
ObjectInspectorand set it in Drill ValueVector. These objects are created in setup.
This way we avoid the switch case evalulation of type for each field in each row.
+ Use one form of Hive data type (TypeInfo) instead of two forms (string based type and type
from ObjectInspector). This simplifies the converter functions such as getMinorTypeFromHiveType,
convertPartitionType, etc.
+ Handle few primitive types that were not handled. Reading complex types is still a TODO.
+ Don't use HiveTextRecordReader as it is not completely implemented (few types are not handled)
+ Remove unused code: HiveInputReader.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e31ef078
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e31ef078
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e31ef078
Branch: refs/heads/master
Commit: e31ef0788ce09b12b03371bc4f4482079c0de365
Parents: fc00bc4
Author: vkorukanti <ve...@gmail.com>
Authored: Thu Jun 12 14:07:40 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:51:46 2014 -0700
----------------------------------------------------------------------
.../exec/store/hive/HiveFieldConverter.java | 199 ++++++++
.../drill/exec/store/hive/HiveInputReader.java | 173 -------
.../drill/exec/store/hive/HiveRecordReader.java | 448 ++++++++++---------
.../exec/store/hive/HiveScanBatchCreator.java | 44 +-
.../exec/store/hive/HiveTextRecordReader.java | 16 +-
5 files changed, 461 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e31ef078/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
new file mode 100644
index 0000000..4ce5e98
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.beust.jcommander.internal.Maps;
+import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableDateVector;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableFloat8Vector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableSmallIntVector;
+import org.apache.drill.exec.vector.NullableTimeStampVector;
+import org.apache.drill.exec.vector.NullableTinyIntVector;
+import org.apache.drill.exec.vector.NullableVarBinaryVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.Text;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.util.Map;
+
+public abstract class HiveFieldConverter {
+
+ public abstract boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex);
+
+ private static Map<PrimitiveCategory, Class< ? extends HiveFieldConverter>> primMap = Maps.newHashMap();
+
+ static {
+ primMap.put(PrimitiveCategory.BINARY, Binary.class);
+ primMap.put(PrimitiveCategory.BOOLEAN, Boolean.class);
+ primMap.put(PrimitiveCategory.BYTE, Byte.class);
+ primMap.put(PrimitiveCategory.DECIMAL, Decimal.class);
+ primMap.put(PrimitiveCategory.DOUBLE, Double.class);
+ primMap.put(PrimitiveCategory.FLOAT, Float.class);
+ primMap.put(PrimitiveCategory.INT, Int.class);
+ primMap.put(PrimitiveCategory.LONG, Long.class);
+ primMap.put(PrimitiveCategory.SHORT, Short.class);
+ primMap.put(PrimitiveCategory.STRING, String.class);
+ primMap.put(PrimitiveCategory.VARCHAR, VarChar.class);
+ primMap.put(PrimitiveCategory.TIMESTAMP, Timestamp.class);
+ primMap.put(PrimitiveCategory.DATE, Date.class);
+ }
+
+
+ public static HiveFieldConverter create(TypeInfo typeInfo) throws IllegalAccessException, InstantiationException {
+ switch (typeInfo.getCategory()) {
+ case PRIMITIVE:
+ final PrimitiveCategory pCat = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
+ Class< ? extends HiveFieldConverter> clazz = primMap.get(pCat);
+ if (clazz != null)
+ return clazz.newInstance();
+
+ HiveRecordReader.throwUnsupportedHiveDataTypeError(pCat.toString());
+ break;
+
+ case LIST:
+ case MAP:
+ case STRUCT:
+ case UNION:
+ default:
+ HiveRecordReader.throwUnsupportedHiveDataTypeError(typeInfo.getCategory().toString());
+ }
+
+ return null;
+ }
+
+ public static class Binary extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableVarBinaryVector) outputVV).getMutator().setSafe(outputIndex, value, 0, value.length);
+ }
+ }
+
+ public static class Boolean extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final boolean value = (boolean) ((BooleanObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableBitVector) outputVV).getMutator().setSafe(outputIndex, value ? 1 : 0);
+ }
+ }
+
+ public static class Byte extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final byte value = (byte) ((ByteObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableTinyIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Decimal extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final HiveDecimal value = ((HiveDecimalObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ final byte[] strBytes = value.toString().getBytes();
+ return ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, strBytes, 0, strBytes.length);
+ }
+ }
+
+ public static class Double extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final double value = (double) ((DoubleObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableFloat8Vector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Float extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final float value = (float) ((FloatObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableFloat4Vector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Int extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final int value = (int) ((IntObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Long extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final long value = (long) ((LongObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableBigIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Short extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final short value = (short) ((ShortObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableSmallIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class String extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final Text value = ((StringObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
+ final byte[] valueBytes = value.getBytes();
+ final int len = value.getLength();
+ return ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, valueBytes, 0, len);
+ }
+ }
+
+ public static class VarChar extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final Text value = ((HiveVarcharObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue).getTextValue();
+ final byte[] valueBytes = value.getBytes();
+ final int valueLen = value.getLength();
+ return ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, valueBytes, 0, valueLen);
+ }
+ }
+
+ public static class Timestamp extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final java.sql.Timestamp value = ((TimestampObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ final DateTime ts = new DateTime(value.getTime()).withZoneRetainFields(DateTimeZone.UTC);
+ return ((NullableTimeStampVector) outputVV).getMutator().setSafe(outputIndex, ts.getMillis());
+ }
+ }
+
+ public static class Date extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final java.sql.Date value = ((DateObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ final DateTime date = new DateTime(value.getTime()).withZoneRetainFields(DateTimeZone.UTC);
+ return ((NullableDateVector) outputVV).getMutator().setSafe(outputIndex, date.getMillis());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e31ef078/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
deleted file mode 100644
index b50ada8..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hive;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.google.common.collect.Lists;
-import com.google.common.io.ByteArrayDataInput;
-import com.google.common.io.ByteArrayDataOutput;
-import com.google.common.io.ByteStreams;
-
-public class HiveInputReader {
- public static void main(String args[]) throws Exception {
-/*
- String[] columnNames = {"n_nationkey", "n_name", "n_regionkey", "n_comment"};
- String[] columnTypes = {"bigint", "string", "bigint", "string"};
-
- List<FieldSchema> cols = Lists.newArrayList();
-
- for (int i = 0; i < columnNames.length; i++) {
- cols.add(new FieldSchema(columnNames[i], columnTypes[i], null));
- }
- String location = "file:///tmp/nation_s";
- String inputFormat = TextInputFormat.class.getCanonicalName();
- String serdeLib = LazySimpleSerDe.class.getCanonicalName();
-// String inputFormat = HiveHBaseTableInputFormat.class.getCanonicalName();
-// String serdeLib = HBaseSerDe.class.getCanonicalName();
- Map<String, String> serdeParams = new HashMap();
-// serdeParams.put("serialization.format", "1");
-// serdeParams.put("hbase.columns.mapping", ":key,f:name,f:regionkey,f:comment");
- serdeParams.put("serialization.format", "|");
- serdeParams.put("field.delim", "|");
-
-
- Map<String, String> tableParams = new HashMap();
- tableParams.put("hbase.table.name", "nation");
- SerDeInfo serDeInfo = new SerDeInfo(null, serdeLib, serdeParams);
- StorageDescriptor storageDescriptor = new StorageDescriptor(cols, location, inputFormat, null, false, -1, serDeInfo, null, null, null);
- Table table = new Table("table", "default", "sphillips", 0, 0, 0, storageDescriptor, new ArrayList<FieldSchema>(), tableParams, null, null, "MANAGED_TABLE");
- Properties properties = MetaStoreUtils.getTableMetadata(table);
- */
-
- HiveConf conf = new HiveConf();
- conf.set("hive.metastore.uris", "thrift://10.10.31.51:9083");
- HiveMetaStoreClient client = new HiveMetaStoreClient(conf);
- Table table = client.getTable("default", "nation");
- Properties properties = MetaStoreUtils.getTableMetadata(table);
-
- Path path = new Path(table.getSd().getLocation());
- JobConf job = new JobConf();
- for (Object obj : properties.keySet()) {
- job.set((String) obj, (String) properties.get(obj));
- }
-// job.set("hbase.zookeeper.quorum", "10.10.31.51");
-// job.set("hbase.zookeeper.property.clientPort", "5181");
- InputFormat f = (InputFormat) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance();
- job.setInputFormat(f.getClass());
- FileInputFormat.addInputPath(job, path);
- InputFormat format = job.getInputFormat();
- SerDe serde = (SerDe) Class.forName(table.getSd().getSerdeInfo().getSerializationLib()).getConstructor().newInstance();
- serde.initialize(job, properties);
- ObjectInspector inspector = serde.getObjectInspector();
- ObjectInspector.Category cat = inspector.getCategory();
- TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(inspector);
- List<String> columns = null;
- List<TypeInfo> colTypes = null;
- List<ObjectInspector> fieldObjectInspectors = Lists.newArrayList();
-
- switch(typeInfo.getCategory()) {
- case STRUCT:
- columns = ((StructTypeInfo) typeInfo).getAllStructFieldNames();
- colTypes = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
- for (int i = 0; i < columns.size(); i++) {
- System.out.print(columns.get(i));
- System.out.print(" ");
- System.out.print(colTypes.get(i));
- }
- System.out.println("");
- for (StructField field : ((StructObjectInspector)inspector).getAllStructFieldRefs()) {
- fieldObjectInspectors.add(field.getFieldObjectInspector());
- }
- }
-
- for (InputSplit split : format.getSplits(job, 1)) {
- String encoded = serializeInputSplit(split);
- System.out.println(encoded);
- InputSplit newSplit = deserializeInputSplit(encoded, split.getClass().getCanonicalName());
- System.out.print("Length: " + newSplit.getLength() + " ");
- System.out.print("Locations: ");
- for (String loc : newSplit.getLocations()) System.out.print(loc + " " );
- System.out.println();
- }
-
- for (InputSplit split : format.getSplits(job, 1)) {
- RecordReader reader = format.getRecordReader(split, job, Reporter.NULL);
- Object key = reader.createKey();
- Object value = reader.createValue();
- int count = 0;
- while (reader.next(key, value)) {
- List<Object> values = ((StructObjectInspector) inspector).getStructFieldsDataAsList(serde.deserialize((Writable) value));
- StructObjectInspector sInsp = (StructObjectInspector) inspector;
- Object obj = sInsp.getStructFieldData(serde.deserialize((Writable) value) , sInsp.getStructFieldRef("n_name"));
- System.out.println(obj);
- /*
- for (Object obj : values) {
- PrimitiveObjectInspector.PrimitiveCategory pCat = ((PrimitiveObjectInspector)fieldObjectInspectors.get(count)).getPrimitiveCategory();
- Object pObj = ((PrimitiveObjectInspector)fieldObjectInspectors.get(count)).getPrimitiveJavaObject(obj);
- System.out.print(pObj + " ");
- }
- */
- System.out.println("");
- }
- }
- }
-
- public static String serializeInputSplit(InputSplit split) throws IOException {
- ByteArrayDataOutput byteArrayOutputStream = ByteStreams.newDataOutput();
- split.write(byteArrayOutputStream);
- return Base64.encodeBase64String(byteArrayOutputStream.toByteArray());
- }
-
- public static InputSplit deserializeInputSplit(String base64, String className) throws Exception {
- InputSplit split;
- if (Class.forName(className) == FileSplit.class) {
- split = new FileSplit((Path) null, 0, 0, (String[])null);
- } else {
- split = (InputSplit) Class.forName(className).getConstructor().newInstance();
- }
- ByteArrayDataInput byteArrayDataInput = ByteStreams.newDataInput(Base64.decodeBase64(base64));
- split.readFields(byteArrayDataInput);
- return split;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e31ef078/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index edd79e6..ac0f036 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -19,20 +19,19 @@ package org.apache.drill.exec.store.hive;
import java.io.IOException;
import java.sql.Timestamp;
-import java.util.Date;
+import java.sql.Date;
import java.util.List;
import java.util.Properties;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
@@ -42,22 +41,12 @@ import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.Float4Vector;
import org.apache.drill.exec.vector.Float8Vector;
import org.apache.drill.exec.vector.IntVector;
-import org.apache.drill.exec.vector.NullableBigIntVector;
-import org.apache.drill.exec.vector.NullableBitVector;
-import org.apache.drill.exec.vector.NullableFloat4Vector;
-import org.apache.drill.exec.vector.NullableFloat8Vector;
-import org.apache.drill.exec.vector.NullableIntVector;
-import org.apache.drill.exec.vector.NullableSmallIntVector;
-import org.apache.drill.exec.vector.NullableTinyIntVector;
-import org.apache.drill.exec.vector.NullableVarBinaryVector;
-import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.apache.drill.exec.vector.NullableTimeStampVector;
import org.apache.drill.exec.vector.TimeStampVector;
-import org.apache.drill.exec.vector.NullableDateVector;
import org.apache.drill.exec.vector.DateVector;
import org.apache.drill.exec.vector.SmallIntVector;
import org.apache.drill.exec.vector.TinyIntVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
import org.apache.hadoop.hive.common.type.HiveDecimal;
@@ -69,12 +58,13 @@ import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
@@ -92,32 +82,34 @@ public class HiveRecordReader implements RecordReader {
protected Partition partition;
protected InputSplit inputSplit;
protected FragmentContext context;
- protected List<SchemaPath> columns;
- protected List<String> columnNames;
- protected List<String> partitionNames = Lists.newArrayList();
+ protected List<SchemaPath> projectedColumns;
+ protected List<String> selectedColumnNames;
+ protected List<TypeInfo> selectedColumnTypes = Lists.newArrayList();
+ protected List<ObjectInspector> selectedColumnObjInspectors = Lists.newArrayList();
+ protected List<HiveFieldConverter> selectedColumnFieldConverters = Lists.newArrayList();
protected List<String> selectedPartitionNames = Lists.newArrayList();
- protected List<String> selectedPartitionTypes = Lists.newArrayList();
- protected List<String> tableColumns;
+ protected List<TypeInfo> selectedPartitionTypes = Lists.newArrayList();
+ protected List<Object> selectedPartitionValues = Lists.newArrayList();
+ protected List<String> tableColumns; // all columns in table (not including partition columns)
protected SerDe serde;
protected StructObjectInspector sInspector;
- protected List<PrimitiveObjectInspector> fieldInspectors = Lists.newArrayList();
- protected List<PrimitiveCategory> primitiveCategories = Lists.newArrayList();
protected Object key, value;
protected org.apache.hadoop.mapred.RecordReader reader;
protected List<ValueVector> vectors = Lists.newArrayList();
protected List<ValueVector> pVectors = Lists.newArrayList();
protected Object redoRecord;
- List<Object> partitionValues = Lists.newArrayList();
protected boolean empty;
protected static final int TARGET_RECORD_COUNT = 4000;
+ protected static final int FIELD_SIZE = 50;
- public HiveRecordReader(Table table, Partition partition, InputSplit inputSplit, List<SchemaPath> columns, FragmentContext context) throws ExecutionSetupException {
+ public HiveRecordReader(Table table, Partition partition, InputSplit inputSplit, List<SchemaPath> projectedColumns,
+ FragmentContext context) throws ExecutionSetupException {
this.table = table;
this.partition = partition;
this.inputSplit = inputSplit;
this.context = context;
- this.columns = columns;
+ this.projectedColumns = projectedColumns;
this.empty = (inputSplit == null && partition == null);
init();
}
@@ -147,6 +139,7 @@ public class HiveRecordReader implements RecordReader {
job.setInputFormat(format.getClass());
List<FieldSchema> partitionKeys = table.getPartitionKeys();
+ List<String> partitionNames = Lists.newArrayList();
for (FieldSchema field : partitionKeys) {
partitionNames.add(field.getName());
}
@@ -158,14 +151,14 @@ public class HiveRecordReader implements RecordReader {
}
sInspector = (StructObjectInspector) oi;
StructTypeInfo sTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(sInspector);
- if (columns == null) {
- columnNames = sTypeInfo.getAllStructFieldNames();
- tableColumns = columnNames;
+ if (projectedColumns == null) {
+ selectedColumnNames = sTypeInfo.getAllStructFieldNames();
+ tableColumns = selectedColumnNames;
} else {
tableColumns = sTypeInfo.getAllStructFieldNames();
List<Integer> columnIds = Lists.newArrayList();
- columnNames = Lists.newArrayList();
- for (SchemaPath field : columns) {
+ selectedColumnNames = Lists.newArrayList();
+ for (SchemaPath field : projectedColumns) {
String columnName = field.getRootSegment().getPath(); //TODO?
if (!tableColumns.contains(columnName)) {
if (partitionNames.contains(columnName)) {
@@ -175,44 +168,46 @@ public class HiveRecordReader implements RecordReader {
}
} else {
columnIds.add(tableColumns.indexOf(columnName));
- columnNames.add(columnName);
+ selectedColumnNames.add(columnName);
}
}
ColumnProjectionUtils.appendReadColumnIDs(job, columnIds);
- ColumnProjectionUtils.appendReadColumnNames(job, columnNames);
+ ColumnProjectionUtils.appendReadColumnNames(job, selectedColumnNames);
}
- for (String columnName : columnNames) {
- ObjectInspector poi = sInspector.getStructFieldRef(columnName).getFieldObjectInspector();
- if(poi.getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UnsupportedOperationException(String.format("%s type not supported", poi.getCategory()));
- }
- PrimitiveObjectInspector pInspector = (PrimitiveObjectInspector) poi;
- fieldInspectors.add(pInspector);
- primitiveCategories.add(pInspector.getPrimitiveCategory());
+
+ for (String columnName : selectedColumnNames) {
+ ObjectInspector fieldOI = sInspector.getStructFieldRef(columnName).getFieldObjectInspector();
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fieldOI.getTypeName());
+
+ selectedColumnObjInspectors.add(fieldOI);
+ selectedColumnTypes.add(typeInfo);
+ selectedColumnFieldConverters.add(HiveFieldConverter.create(typeInfo));
}
- if (columns == null) {
+ if (projectedColumns == null) {
selectedPartitionNames = partitionNames;
}
for (int i = 0; i < table.getPartitionKeys().size(); i++) {
FieldSchema field = table.getPartitionKeys().get(i);
if (selectedPartitionNames.contains(field.getName())) {
- selectedPartitionTypes.add(field.getType());
+ TypeInfo pType = TypeInfoUtils.getTypeInfoFromTypeString(field.getType());
+ selectedPartitionTypes.add(pType);
+
if (partition != null) {
- partitionValues.add(convertPartitionType(field.getType(), partition.getValues().get(i)));
+ selectedPartitionValues.add(convertPartitionType(pType, partition.getValues().get(i)));
}
}
}
- } catch (SerDeException e) {
- throw new ExecutionSetupException(e);
+ } catch (Exception e) {
+ throw new ExecutionSetupException("Failure while initializing HiveRecordReader: " + e.getMessage(), e);
}
if (!empty) {
try {
reader = format.getRecordReader(inputSplit, job, Reporter.NULL);
} catch (IOException e) {
- throw new ExecutionSetupException("Failed to get Recordreader", e);
+ throw new ExecutionSetupException("Failed to get o.a.hadoop.mapred.RecordReader from Hive InputFormat", e);
}
key = reader.createKey();
value = reader.createValue();
@@ -222,16 +217,16 @@ public class HiveRecordReader implements RecordReader {
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
try {
- for (int i = 0; i < columnNames.size(); i++) {
- PrimitiveCategory pCat = primitiveCategories.get(i);
- MajorType type = getMajorType(pCat);
- MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(columnNames.get(i)), type);
- ValueVector vv = output.addField(field, (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()));
- vectors.add(vv);
+ for (int i = 0; i < selectedColumnNames.size(); i++) {
+ MajorType type = Types.optional(getMinorTypeFromHiveTypeInfo(selectedColumnTypes.get(i)));
+ MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(selectedColumnNames.get(i)), type);
+ Class vvClass = TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode());
+ vectors.add(output.addField(field, vvClass));
}
+
for (int i = 0; i < selectedPartitionNames.size(); i++) {
- String type = selectedPartitionTypes.get(i);
- MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(selectedPartitionNames.get(i)), Types.getMajorTypeFromName(type));
+ MajorType type = Types.required(getMinorTypeFromHiveTypeInfo(selectedPartitionTypes.get(i)));
+ MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(selectedPartitionNames.get(i)), type);
Class vvClass = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getDataMode());
pVectors.add(output.addField(field, vvClass));
}
@@ -240,17 +235,152 @@ public class HiveRecordReader implements RecordReader {
}
}
+ @Override
+ public int next() {
+ if (empty) {
+ return 0;
+ }
+
+ for (ValueVector vv : vectors) {
+ VectorAllocator.getAllocator(vv, FIELD_SIZE).alloc(TARGET_RECORD_COUNT);
+ }
+
+ try {
+ int recordCount = 0;
+
+ if (redoRecord != null) {
+ // Try writing the record that didn't fit into the last RecordBatch
+ Object deSerializedValue = serde.deserialize((Writable) redoRecord);
+ boolean status = readHiveRecordAndInsertIntoRecordBatch(deSerializedValue, recordCount);
+ if (!status) {
+ throw new DrillRuntimeException("Current record is too big to fit into allocated ValueVector buffer");
+ }
+ redoRecord = null;
+ recordCount++;
+ }
+
+ while (recordCount < TARGET_RECORD_COUNT && reader.next(key, value)) {
+ Object deSerializedValue = serde.deserialize((Writable) value);
+ boolean status = readHiveRecordAndInsertIntoRecordBatch(deSerializedValue, recordCount);
+ if (!status) {
+ redoRecord = value;
+ setValueCountAndPopulatePartitionVectors(recordCount);
+ return recordCount;
+ }
+ recordCount++;
+ }
+
+ setValueCountAndPopulatePartitionVectors(recordCount);
+ return recordCount;
+ } catch (IOException | SerDeException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ private boolean readHiveRecordAndInsertIntoRecordBatch(Object deSerializedValue, int outputRecordIndex) {
+ boolean success;
+ for (int i = 0; i < selectedColumnNames.size(); i++) {
+ String columnName = selectedColumnNames.get(i);
+ Object hiveValue = sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName));
+
+ if (hiveValue != null) {
+ success = selectedColumnFieldConverters.get(i).setSafeValue(selectedColumnObjInspectors.get(i), hiveValue,
+ vectors.get(i), outputRecordIndex);
+
+ if (!success) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ private void setValueCountAndPopulatePartitionVectors(int recordCount) {
+ for (ValueVector v : vectors) {
+ v.getMutator().setValueCount(recordCount);
+ }
+
+ if (partition != null) {
+ populatePartitionVectors(recordCount);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ }
+
+ public static MinorType getMinorTypeFromHivePrimitiveTypeInfo(PrimitiveTypeInfo primitiveTypeInfo) {
+ switch(primitiveTypeInfo.getPrimitiveCategory()) {
+ case BINARY:
+ return TypeProtos.MinorType.VARBINARY;
+ case BOOLEAN:
+ return TypeProtos.MinorType.BIT;
+ case BYTE:
+ return TypeProtos.MinorType.TINYINT;
+ case DECIMAL:
+ return TypeProtos.MinorType.VARCHAR;
+ case DOUBLE:
+ return TypeProtos.MinorType.FLOAT8;
+ case FLOAT:
+ return TypeProtos.MinorType.FLOAT4;
+ case INT:
+ return TypeProtos.MinorType.INT;
+ case LONG:
+ return TypeProtos.MinorType.BIGINT;
+ case SHORT:
+ return TypeProtos.MinorType.SMALLINT;
+ case STRING:
+ case VARCHAR:
+ return TypeProtos.MinorType.VARCHAR;
+ case TIMESTAMP:
+ return TypeProtos.MinorType.TIMESTAMP;
+ case DATE:
+ return TypeProtos.MinorType.DATE;
+ }
+
+ throwUnsupportedHiveDataTypeError(primitiveTypeInfo.getPrimitiveCategory().toString());
+ return null;
+ }
+
+ public static MinorType getMinorTypeFromHiveTypeInfo(TypeInfo typeInfo) {
+ switch (typeInfo.getCategory()) {
+ case PRIMITIVE:
+ return getMinorTypeFromHivePrimitiveTypeInfo(((PrimitiveTypeInfo) typeInfo));
+
+ case LIST:
+ case MAP:
+ case STRUCT:
+ case UNION:
+ default:
+ throwUnsupportedHiveDataTypeError(typeInfo.getCategory().toString());
+ }
+
+ return null;
+ }
+
protected void populatePartitionVectors(int recordCount) {
for (int i = 0; i < pVectors.size(); i++) {
int size = 50;
ValueVector vector = pVectors.get(i);
- Object val = partitionValues.get(i);
- if (selectedPartitionTypes.get(i).equals("string") || selectedPartitionTypes.get(i).equals("binary")) {
- size = ((byte[]) partitionValues.get(i)).length;
+ Object val = selectedPartitionValues.get(i);
+ PrimitiveCategory pCat = ((PrimitiveTypeInfo)selectedPartitionTypes.get(i)).getPrimitiveCategory();
+ if (pCat == PrimitiveCategory.BINARY || pCat == PrimitiveCategory.STRING || pCat == PrimitiveCategory.VARCHAR) {
+ size = ((byte[]) selectedPartitionValues.get(i)).length;
}
+
VectorAllocator.getAllocator(vector, size).alloc(recordCount);
- switch(selectedPartitionTypes.get(i)) {
- case "boolean": {
+
+ switch(pCat) {
+ case BINARY: {
+ VarBinaryVector v = (VarBinaryVector) vector;
+ byte[] value = (byte[]) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().setSafe(j, value);
+ }
+ break;
+ }
+ case BOOLEAN: {
BitVector v = (BitVector) vector;
Boolean value = (Boolean) val;
for (int j = 0; j < recordCount; j++) {
@@ -258,7 +388,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "tinyint": {
+ case BYTE: {
TinyIntVector v = (TinyIntVector) vector;
byte value = (byte) val;
for (int j = 0; j < recordCount; j++) {
@@ -266,7 +396,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "double": {
+ case DOUBLE: {
Float8Vector v = (Float8Vector) vector;
double value = (double) val;
for (int j = 0; j < recordCount; j++) {
@@ -274,7 +404,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "float": {
+ case FLOAT: {
Float4Vector v = (Float4Vector) vector;
float value = (float) val;
for (int j = 0; j < recordCount; j++) {
@@ -282,7 +412,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "int": {
+ case INT: {
IntVector v = (IntVector) vector;
int value = (int) val;
for (int j = 0; j < recordCount; j++) {
@@ -290,7 +420,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "bigint": {
+ case LONG: {
BigIntVector v = (BigIntVector) vector;
long value = (long) val;
for (int j = 0; j < recordCount; j++) {
@@ -298,7 +428,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "smallint": {
+ case SHORT: {
SmallIntVector v = (SmallIntVector) vector;
short value = (short) val;
for (int j = 0; j < recordCount; j++) {
@@ -306,7 +436,8 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "string": {
+ case VARCHAR:
+ case STRING: {
VarCharVector v = (VarCharVector) vector;
byte[] value = (byte[]) val;
for (int j = 0; j < recordCount; j++) {
@@ -314,7 +445,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "timestamp": {
+ case TIMESTAMP: {
TimeStampVector v = (TimeStampVector) vector;
DateTime ts = new DateTime(((Timestamp) val).getTime()).withZoneRetainFields(DateTimeZone.UTC);
long value = ts.getMillis();
@@ -323,7 +454,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "date": {
+ case DATE: {
DateVector v = (DateVector) vector;
DateTime date = new DateTime(((Date)val).getTime()).withZoneRetainFields(DateTimeZone.UTC);
long value = date.getMillis();
@@ -332,7 +463,7 @@ public class HiveRecordReader implements RecordReader {
}
break;
}
- case "decimal": {
+ case DECIMAL: {
VarCharVector v = (VarCharVector) vector;
byte[] value = ((HiveDecimal) val).toString().getBytes();
for (int j = 0; j < recordCount; j++) {
@@ -341,174 +472,61 @@ public class HiveRecordReader implements RecordReader {
break;
}
default:
- throw new UnsupportedOperationException("Could not determine type: " + selectedPartitionTypes.get(i));
+ throwUnsupportedHiveDataTypeError(pCat.toString());
}
vector.getMutator().setValueCount(recordCount);
}
}
- private Object convertPartitionType(String type, String value) {
- switch (type) {
- case "boolean":
- return Boolean.parseBoolean(value);
- case "tinyint":
- return Byte.parseByte(value);
- case "double":
- return Double.parseDouble(value);
- case "float":
- return Float.parseFloat(value);
- case "int":
- return Integer.parseInt(value);
- case "bigint":
- return Long.parseLong(value);
- case "smallint":
- return Short.parseShort(value);
- case "string":
- return value.getBytes();
- default:
- throw new UnsupportedOperationException("Could not determine type: " + type);
+ /** Partition value is received in string format. Convert it into appropriate object based on the type. */
+ private Object convertPartitionType(TypeInfo typeInfo, String value) {
+ if (typeInfo.getCategory() != Category.PRIMITIVE) {
+ // In Hive only primitive types are allowed as partition column types.
+ throw new DrillRuntimeException("Non-Primitive types are not allowed as partition column type in Hive, " +
+ "but received one: " + typeInfo.getCategory());
}
- }
- public static TypeProtos.MajorType getMajorType(PrimitiveCategory pCat) {
- switch(pCat) {
+ PrimitiveCategory pCat = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
+ switch (pCat) {
case BINARY:
- return Types.optional(TypeProtos.MinorType.VARBINARY);
+ return value.getBytes();
case BOOLEAN:
- return Types.optional(TypeProtos.MinorType.BIT);
+ return Boolean.parseBoolean(value);
case BYTE:
- return Types.optional(TypeProtos.MinorType.TINYINT);
+ return Byte.parseByte(value);
case DECIMAL:
- return Types.optional(TypeProtos.MinorType.VARCHAR);
+ return new HiveDecimal(value);
case DOUBLE:
- return Types.optional(TypeProtos.MinorType.FLOAT8);
+ return Double.parseDouble(value);
case FLOAT:
- return Types.optional(TypeProtos.MinorType.FLOAT4);
+ return Float.parseFloat(value);
case INT:
- return Types.optional(TypeProtos.MinorType.INT);
+ return Integer.parseInt(value);
case LONG:
- return Types.optional(TypeProtos.MinorType.BIGINT);
+ return Long.parseLong(value);
case SHORT:
- return Types.optional(TypeProtos.MinorType.SMALLINT);
+ return Short.parseShort(value);
case STRING:
- return Types.optional(TypeProtos.MinorType.VARCHAR);
+ case VARCHAR:
+ return value.getBytes();
case TIMESTAMP:
- return Types.optional(TypeProtos.MinorType.TIMESTAMP);
+ return Timestamp.valueOf(value);
case DATE:
- return Types.optional(TypeProtos.MinorType.DATE);
- default:
- throw new UnsupportedOperationException("Could not determine type: " + pCat);
+ return Date.valueOf(value);
}
- }
- public boolean setValue(PrimitiveCategory pCat, ValueVector vv, int index, Object fieldValue) {
- switch(pCat) {
- case BINARY:
- return ((NullableVarBinaryVector) vv).getMutator().setSafe(index, (byte[]) fieldValue, 0, ((byte[]) fieldValue).length);
- case BOOLEAN:
- boolean isSet = (boolean) fieldValue;
- return ((NullableBitVector) vv).getMutator().setSafe(index, isSet ? 1 : 0 );
- case BYTE:
- return ((NullableTinyIntVector) vv).getMutator().setSafe(index, (byte) fieldValue);
- case DECIMAL:
- String value = ((HiveDecimal) fieldValue).toString();
- int strLen = value.length();
- byte[] strBytes = value.getBytes();
- return ((NullableVarCharVector) vv).getMutator().setSafe(index, strBytes, 0, strLen);
- case DOUBLE:
- return ((NullableFloat8Vector) vv).getMutator().setSafe(index, (double) fieldValue);
- case FLOAT:
- return ((NullableFloat4Vector) vv).getMutator().setSafe(index, (float) fieldValue);
- case INT:
- return ((NullableIntVector) vv).getMutator().setSafe(index, (int) fieldValue);
- case LONG:
- return ((NullableBigIntVector) vv).getMutator().setSafe(index, (long) fieldValue);
- case SHORT:
- return ((NullableSmallIntVector) vv).getMutator().setSafe(index, (short) fieldValue);
- case STRING:
- int len = ((Text) fieldValue).getLength();
- byte[] bytes = ((Text) fieldValue).getBytes();
- return ((NullableVarCharVector) vv).getMutator().setSafe(index, bytes, 0, len);
- case TIMESTAMP:
- DateTime ts = new DateTime(((Timestamp) fieldValue).getTime()).withZoneRetainFields(DateTimeZone.UTC);
- return ((NullableTimeStampVector)vv).getMutator().setSafe(index, ts.getMillis());
- case DATE:
- DateTime date = new DateTime(((Date) fieldValue).getTime()).withZoneRetainFields(DateTimeZone.UTC);
- return ((NullableDateVector)vv).getMutator().setSafe(index, date.getMillis());
- default:
- throw new UnsupportedOperationException("Could not determine type");
- }
+ throwUnsupportedHiveDataTypeError(pCat.toString());
+ return null;
}
- @Override
- public int next() {
- if (empty) {
- return 0;
- }
+ public static void throwUnsupportedHiveDataTypeError(String unsupportedType) {
+ StringBuilder errMsg = new StringBuilder();
+ errMsg.append(String.format("Unsupported Hive data type %s. ", unsupportedType));
+ errMsg.append(System.getProperty("line.separator"));
+ errMsg.append("Following Hive data types are supported in Drill for querying: ");
+ errMsg.append(
+ "BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, TIMESTAMP, BINARY, DECIMAL, STRING, and VARCHAR");
- for (ValueVector vv : vectors) {
- VectorAllocator.getAllocator(vv, 50).alloc(TARGET_RECORD_COUNT);
- }
- try {
- int recordCount = 0;
- if (redoRecord != null) {
- Object deSerializedValue = serde.deserialize((Writable) redoRecord);
- for (int i = 0; i < columnNames.size(); i++) {
- Object obj;
- String columnName = columnNames.get(i);
- if (primitiveCategories.get(i) == PrimitiveCategory.STRING) {
- obj = fieldInspectors.get(i).getPrimitiveWritableObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
- } else {
- obj = fieldInspectors.get(i).getPrimitiveJavaObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
- }
- boolean success = true;
- if( obj != null ) {
- success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount, obj);
- }
- if (!success) {
- throw new DrillRuntimeException(String.format("Failed to write value for column %s", columnName));
- }
- }
- redoRecord = null;
- recordCount++;
- }
- while (recordCount < TARGET_RECORD_COUNT && reader.next(key, value)) {
- Object deSerializedValue = serde.deserialize((Writable) value);
- for (int i = 0; i < columnNames.size(); i++) {
- Object obj;
- String columnName = columnNames.get(i);
- if (primitiveCategories.get(i) == PrimitiveCategory.STRING) {
- obj = fieldInspectors.get(i).getPrimitiveWritableObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
- } else {
- obj = fieldInspectors.get(i).getPrimitiveJavaObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
- }
- boolean success = true;
- if( obj != null ) {
- success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount, obj);
- }
- if (!success) {
- redoRecord = value;
- if (partition != null) populatePartitionVectors(recordCount);
- for (ValueVector v : vectors) {
- v.getMutator().setValueCount(recordCount);
- }
- if (partition != null) populatePartitionVectors(recordCount);
- return recordCount;
- }
- }
- recordCount++;
- }
- for (ValueVector v : vectors) {
- v.getMutator().setValueCount(recordCount);
- }
- if (partition != null) populatePartitionVectors(recordCount);
- return recordCount;
- } catch (IOException | SerDeException e) {
- throw new DrillRuntimeException(e);
- }
- }
-
- @Override
- public void cleanup() {
+ throw new RuntimeException(errMsg.toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e31ef078/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
index 8914db2..6e540ad 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
@@ -40,32 +40,26 @@ public class HiveScanBatchCreator implements BatchCreator<HiveSubScan> {
Table table = config.getTable();
List<InputSplit> splits = config.getInputSplits();
List<Partition> partitions = config.getPartitions();
- if (partitions == null || partitions.size() == 0) {
- if (table.getSd().getInputFormat().equals(TextInputFormat.class.getCanonicalName()) &&
- table.getSd().getSerdeInfo().getSerializationLib().equals(LazySimpleSerDe.class.getCanonicalName()) &&
- config.getColumns() != null) {
- for (InputSplit split : splits) {
- readers.add(new HiveTextRecordReader(table, null, split, config.getColumns(), context));
- }
- } else {
- for (InputSplit split : splits) {
- readers.add(new HiveRecordReader(table, null, split, config.getColumns(), context));
- }
- }
- } else {
- int i = 0;
- if (table.getSd().getInputFormat().equals(TextInputFormat.class.getCanonicalName()) &&
- table.getSd().getSerdeInfo().getSerializationLib().equals(LazySimpleSerDe.class.getCanonicalName()) &&
- config.getColumns() != null) {
- for (InputSplit split : splits) {
- readers.add(new HiveTextRecordReader(table, partitions.get(i++), split, config.getColumns(), context));
- }
- } else {
- for (InputSplit split : splits) {
- readers.add(new HiveRecordReader(config.getTable(), partitions.get(i++), split, config.getColumns(), context));
- }
+ boolean hasPartitions = (partitions != null && partitions.size() > 0);
+ int i = 0;
+
+ // Native hive text record reader doesn't handle all types currently. For now use HiveRecordReader which uses
+ // Hive InputFormat and SerDe classes to read the data.
+ //if (table.getSd().getInputFormat().equals(TextInputFormat.class.getCanonicalName()) &&
+ // table.getSd().getSerdeInfo().getSerializationLib().equals(LazySimpleSerDe.class.getCanonicalName()) &&
+ // config.getColumns() != null) {
+ // for (InputSplit split : splits) {
+ // readers.add(new HiveTextRecordReader(table,
+ // (hasPartitions ? partitions.get(i++) : null),
+ // split, config.getColumns(), context));
+ // }
+ //} else {
+ for (InputSplit split : splits) {
+ readers.add(new HiveRecordReader(table,
+ (hasPartitions ? partitions.get(i++) : null),
+ split, config.getColumns(), context));
}
- }
+ //}
// If there are no readers created (which is possible when the table is empty), create an empty RecordReader to
// output the schema
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e31ef078/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
index b72acf5..116603c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
@@ -37,14 +37,18 @@ import org.apache.hadoop.mapred.InputSplit;
import com.google.common.collect.Lists;
+/**
+ * Note: Native hive text record reader is not complete in implementation. For now use
+ * {@link org.apache.drill.exec.store.hive.HiveRecordReader}.
+ */
public class HiveTextRecordReader extends HiveRecordReader {
public final byte delimiter;
public final List<Integer> columnIds;
private final int numCols;
- public HiveTextRecordReader(Table table, Partition partition, InputSplit inputSplit, List<SchemaPath> columns, FragmentContext context) throws ExecutionSetupException {
- super(table, partition, inputSplit, columns, context);
+ public HiveTextRecordReader(Table table, Partition partition, InputSplit inputSplit, List<SchemaPath> projectedColumns, FragmentContext context) throws ExecutionSetupException {
+ super(table, partition, inputSplit, projectedColumns, context);
String d = table.getSd().getSerdeInfo().getParameters().get("field.delim");
if (d != null) {
delimiter = d.getBytes()[0];
@@ -54,7 +58,7 @@ public class HiveTextRecordReader extends HiveRecordReader {
assert delimiter > 0;
List<Integer> ids = Lists.newArrayList();
for (int i = 0; i < tableColumns.size(); i++) {
- if (columnNames.contains(tableColumns.get(i))) {
+ if (selectedColumnNames.contains(tableColumns.get(i))) {
ids.add(i);
}
}
@@ -133,9 +137,9 @@ public class HiveTextRecordReader extends HiveRecordReader {
}
}
for (int id : columnIds) {
- boolean success = setValue(primitiveCategories.get(id), vectors.get(id), recordCount, bytes, delimPositions[id]);
+ boolean success = false; // setValue(primitiveCategories.get(id), vectors.get(id), recordCount, bytes, delimPositions[id]);
if (!success) {
- throw new DrillRuntimeException(String.format("Failed to write value for column %s", columnNames.get(id)));
+ throw new DrillRuntimeException(String.format("Failed to write value for column %s", selectedColumnNames.get(id)));
}
}
@@ -154,7 +158,7 @@ public class HiveTextRecordReader extends HiveRecordReader {
}
for (int i = 0; i < columnIds.size(); i++) {
int id = columnIds.get(i);
- boolean success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount, bytes, delimPositions[id] + 1);
+ boolean success = false; // setValue(primitiveCategories.get(i), vectors.get(i), recordCount, bytes, delimPositions[id] + 1);
if (!success) {
redoRecord = value;
if (partition != null) populatePartitionVectors(recordCount);
[31/32] git commit: DRILL-937: Optimize boolean and/or operator with
short circuit evaluation in run-time generated code.
Posted by ja...@apache.org.
DRILL-937: Optimize boolean and/or operator with short circuit evaluation in run-time generated code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/38dde9e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/38dde9e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/38dde9e9
Branch: refs/heads/master
Commit: 38dde9e9d780aa3dba6faa1c44d6669e7f4bb202
Parents: 1568fc6
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Sun Jun 15 14:28:12 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 13:50:49 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/expr/ClassGenerator.java | 16 +++
.../drill/exec/expr/EvaluationVisitor.java | 136 +++++++++++++++++++
.../exec/expr/annotations/FunctionTemplate.java | 3 +-
.../exec/expr/fn/DrillSCBooleanOPHolder.java | 35 +++++
.../drill/exec/expr/fn/FunctionConverter.java | 6 +
.../drill/exec/expr/fn/impl/BitFunctions.java | 4 +-
6 files changed, 197 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/38dde9e9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
index ab3c307..7b1832c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
@@ -49,6 +49,7 @@ import com.sun.codemodel.JExpr;
import com.sun.codemodel.JExpression;
import com.sun.codemodel.JFieldRef;
import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JLabel;
import com.sun.codemodel.JMethod;
import com.sun.codemodel.JMod;
import com.sun.codemodel.JType;
@@ -76,6 +77,7 @@ public class ClassGenerator<T>{
private final JCodeModel model;
private int index = 0;
+ private int labelIndex = 0;
private MappingSet mappings;
public static MappingSet getDefaultMapping(){
@@ -149,6 +151,20 @@ public class ClassGenerator<T>{
return getBlock(getCurrentMapping().getMethodName(BlockType.CLEANUP));
}
+ public void nestEvalBlock(JBlock block) {
+ String methodName = getCurrentMapping().getMethodName(BlockType.EVAL);
+ this.blocks[sig.get(methodName)].addLast(block);
+ }
+
+ public void unNestEvalBlock() {
+ String methodName = getCurrentMapping().getMethodName(BlockType.EVAL);
+ this.blocks[sig.get(methodName)].removeLast();
+ }
+
+ public JLabel getEvalBlockLabel (String prefix) {
+ return getEvalBlock().label(prefix + labelIndex ++);
+ }
+
public JVar declareVectorValueSetupAndMember(String batchName, TypedFieldId fieldId){
return declareVectorValueSetupAndMember( DirectExpression.direct(batchName), fieldId);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/38dde9e9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 31cc563..efc7259 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -55,6 +55,7 @@ import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.expr.ClassGenerator.BlockType;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
+import org.apache.drill.exec.expr.fn.DrillSCBooleanOPHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.fn.HiveFuncHolder;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
@@ -104,6 +105,15 @@ public class EvaluationVisitor {
if (holderExpr instanceof DrillFuncHolderExpr) {
DrillFuncHolder holder = ((DrillFuncHolderExpr) holderExpr).getHolder();
+
+ if (holder instanceof DrillSCBooleanOPHolder && holderExpr.getName().equals("booleanAnd")) {
+ return visitBooleanAnd(holderExpr, generator);
+ }
+
+ if (holder instanceof DrillSCBooleanOPHolder && holderExpr.getName().equals("booleanOr")) {
+ return visitBooleanOr(holderExpr, generator);
+ }
+
JVar[] workspaceVars = holder.renderStart(generator, null);
if (holder.isNested())
@@ -592,6 +602,132 @@ public class EvaluationVisitor {
FunctionCall fc = new FunctionCall(convertFunctionName, newArgs, e.getPosition());
return fc.accept(this, value);
}
+
+ private HoldingContainer visitBooleanAnd(FunctionHolderExpression holderExpr,
+ ClassGenerator<?> generator) {
+
+ HoldingContainer out = generator.declare(holderExpr.getMajorType());
+
+ JLabel label = generator.getEvalBlockLabel("AndOP");
+ JBlock eval = generator.getEvalBlock().block(); // enter into nested block
+ generator.nestEvalBlock(eval);
+
+ HoldingContainer arg = null;
+
+ JExpression e = null;
+
+ // value of boolean "and" when one side is null
+ // p q p and q
+ // true null null
+ // false null false
+ // null true null
+ // null false false
+ // null null null
+ for (int i = 0; i < holderExpr.args.size(); i++) {
+ arg = holderExpr.args.get(i).accept(this, generator);
+
+ JBlock earlyExit = null;
+ if (arg.isOptional()) {
+ earlyExit = eval._if(arg.getIsSet().eq(JExpr.lit(1)).cand(arg.getValue().ne(JExpr.lit(1))))._then();
+ if(e == null){
+ e = arg.getIsSet();
+ }else{
+ e = e.mul(arg.getIsSet());
+ }
+ } else {
+ earlyExit = eval._if(arg.getValue().ne(JExpr.lit(1)))._then();
+ }
+
+ if (out.isOptional()) {
+ earlyExit.assign(out.getIsSet(), JExpr.lit(1));
+ }
+
+ earlyExit.assign(out.getValue(), JExpr.lit(0));
+ earlyExit._break(label);
+ }
+
+ if (out.isOptional()) {
+ assert (e != null);
+
+ JConditional notSetJC = eval._if(e.eq(JExpr.lit(0)));
+ notSetJC._then().assign(out.getIsSet(), JExpr.lit(0));
+
+ JBlock setBlock = notSetJC._else().block();
+ setBlock.assign(out.getIsSet(), JExpr.lit(1));
+ setBlock.assign(out.getValue(), JExpr.lit(1));
+ } else {
+ assert (e == null);
+ eval.assign(out.getValue(), JExpr.lit(1)) ;
+ }
+
+ generator.unNestEvalBlock(); // exit from nested block
+
+ return out;
+ }
+
+ private HoldingContainer visitBooleanOr(FunctionHolderExpression holderExpr,
+ ClassGenerator<?> generator) {
+
+ HoldingContainer out = generator.declare(holderExpr.getMajorType());
+
+ JLabel label = generator.getEvalBlockLabel("OrOP");
+ JBlock eval = generator.getEvalBlock().block();
+ generator.nestEvalBlock(eval); // enter into nested block.
+
+ HoldingContainer arg = null;
+
+ JExpression e = null;
+
+ // value of boolean "or" when one side is null
+ // p q p and q
+ // true null true
+ // false null null
+ // null true true
+ // null false null
+ // null null null
+
+ for (int i = 0; i < holderExpr.args.size(); i++) {
+ arg = holderExpr.args.get(i).accept(this, generator);
+
+ JBlock earlyExit = null;
+ if (arg.isOptional()) {
+ earlyExit = eval._if(arg.getIsSet().eq(JExpr.lit(1)).cand(arg.getValue().eq(JExpr.lit(1))))._then();
+ if(e == null){
+ e = arg.getIsSet();
+ }else{
+ e = e.mul(arg.getIsSet());
+ }
+ } else {
+ earlyExit = eval._if(arg.getValue().eq(JExpr.lit(1)))._then();
+ }
+
+ if (out.isOptional()) {
+ earlyExit.assign(out.getIsSet(), JExpr.lit(1));
+ }
+
+ earlyExit.assign(out.getValue(), JExpr.lit(1));
+ earlyExit._break(label);
+ }
+
+ if (out.isOptional()) {
+ assert (e != null);
+
+ JConditional notSetJC = eval._if(e.eq(JExpr.lit(0)));
+ notSetJC._then().assign(out.getIsSet(), JExpr.lit(0));
+
+ JBlock setBlock = notSetJC._else().block();
+ setBlock.assign(out.getIsSet(), JExpr.lit(1));
+ setBlock.assign(out.getValue(), JExpr.lit(0));
+ } else {
+ assert (e == null);
+ eval.assign(out.getValue(), JExpr.lit(0)) ;
+ }
+
+ generator.unNestEvalBlock(); // exit from nested block.
+
+ return out;
+ }
+
}
private class ConstantFilter extends EvalVisitor {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/38dde9e9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
index 9d7091d..b364a4a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
@@ -61,6 +61,7 @@ public @interface FunctionTemplate {
DECIMAL_CAST,
DECIMAL_DIV_SCALE,
DECIMAL_SET_SCALE,
- DECIMAL_ZERO_SCALE;
+ DECIMAL_ZERO_SCALE,
+ SC_BOOLEAN_OPERATOR
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/38dde9e9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSCBooleanOPHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSCBooleanOPHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSCBooleanOPHolder.java
new file mode 100644
index 0000000..d6d7037
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSCBooleanOPHolder.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.fn;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+
+public class DrillSCBooleanOPHolder extends DrillSimpleFuncHolder{
+
+ public DrillSCBooleanOPHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
+ String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
+ Map<String, String> methods, List<String> imports) {
+ super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/38dde9e9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 75c3cbb..2107421 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -235,6 +235,12 @@ public class FunctionConverter {
template.isBinaryCommutative(),
template.isRandom(), registeredNames,
ps, outputField, works, methods, imports);
+ case SC_BOOLEAN_OPERATOR:
+ return new DrillSCBooleanOPHolder(template.scope(), template.nulls(),
+ template.isBinaryCommutative(),
+ template.isRandom(), registeredNames,
+ ps, outputField, works, methods, imports);
+
case DECIMAL_MAX_SCALE:
return new DrillDecimalMaxScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/38dde9e9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
index 1ea4988..3ffc682 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.record.RecordBatch;
public class BitFunctions {
- @FunctionTemplate(names = {"booleanOr", "or", "||"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"booleanOr", "or", "||"}, scope = FunctionScope.SC_BOOLEAN_OPERATOR, nulls = NullHandling.NULL_IF_NULL)
public static class BitOr implements DrillSimpleFunc {
@Param BitHolder left;
@@ -43,7 +43,7 @@ public class BitFunctions {
}
}
- @FunctionTemplate(names = {"booleanAnd", "and", "&&"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"booleanAnd", "and", "&&"}, scope = FunctionScope.SC_BOOLEAN_OPERATOR, nulls = NullHandling.NULL_IF_NULL)
public static class BitAnd implements DrillSimpleFunc {
@Param BitHolder left;
[19/32] git commit: DRILL-727: During logical transformation phase
use the Project's traits instead of input traits when creating a
DrillProjectRel.
Posted by ja...@apache.org.
DRILL-727: During logical transformation phase use the Project's traits instead of input traits when creating a DrillProjectRel.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ebe49ea2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ebe49ea2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ebe49ea2
Branch: refs/heads/master
Commit: ebe49ea2b606bb610c5242e61b7e5c61dd86e62a
Parents: 0648c9e
Author: Aman Sinha <as...@maprtech.com>
Authored: Thu Jun 12 19:04:04 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:01 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/planner/logical/DrillProjectRule.java | 4 ++--
.../src/test/java/org/apache/drill/TestExampleQueries.java | 8 ++++++++
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ebe49ea2/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
index 109588c..7cef038 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
@@ -38,9 +38,9 @@ public class DrillProjectRule extends RelOptRule {
public void onMatch(RelOptRuleCall call) {
final ProjectRel project = (ProjectRel) call.rel(0);
final RelNode input = project.getChild();
- //final RelTraitSet traits = project.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
+ final RelTraitSet traits = project.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.DRILL_LOGICAL));
- call.transformTo(new DrillProjectRel(project.getCluster(), convertedInput.getTraitSet(), convertedInput, project.getProjects(), project
+ call.transformTo(new DrillProjectRel(project.getCluster(), traits, convertedInput, project.getProjects(), project
.getRowType()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ebe49ea2/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 783c562..59556b3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -39,6 +39,7 @@ public class TestExampleQueries extends BaseTestQuery{
test("select count(*) from cp.`customer.json` limit 1");
}
+
@Test
public void testCaseReturnValueVarChar() throws Exception{
test("select case when employee_id < 1000 then 'ABC' else 'DEF' end from cp.`employee.json` limit 5");
@@ -195,4 +196,11 @@ public class TestExampleQueries extends BaseTestQuery{
test("select cast(NON_EXIST_COL as varchar(10)) from cp.`employee.json` limit 2; ");
}
+ @Test // Bugs DRILL-727, DRILL-940
+ public void testOrderByDiffColumn() throws Exception {
+ test("select r_name from cp.`tpch/region.parquet` order by r_regionkey");
+ test("select r_name from cp.`tpch/region.parquet` order by r_name, r_regionkey");
+ test("select cast(r_name as varchar(20)) from cp.`tpch/region.parquet` order by r_name");
+ }
+
}
[07/32] git commit: DRILL-735: Add a unittest to test reading from
Hive table with all supported data types of partitions and fields.
Posted by ja...@apache.org.
DRILL-735: Add a unittest to test reading from Hive table with all supported data types of partitions and fields.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/3bcead76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/3bcead76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/3bcead76
Branch: refs/heads/master
Commit: 3bcead76387e0cc63d2dde04bce28db5b1cf0942
Parents: c3dfe37
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jun 13 14:44:12 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:53:13 2014 -0700
----------------------------------------------------------------------
.../exec/store/hive/HiveTestDataGenerator.java | 76 +++++++++++++++++---
.../apache/drill/jdbc/test/TestHiveStorage.java | 52 +++++++++++++-
.../apache/drill/jdbc/test/TestMetadataDDL.java | 10 +--
3 files changed, 123 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3bcead76/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index f914661..e051abb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -90,15 +90,75 @@ public class HiveTestDataGenerator {
// create a Hive table that has columns with data types which are supported for reading in Drill.
testDataFile = generateAllTypesDataFile();
- executeQuery("CREATE TABLE IF NOT EXISTS allReadSupportedHiveDataTypes (c1 INT, c2 BOOLEAN, c3 DOUBLE, c4 STRING, " +
- "c9 TINYINT, c10 SMALLINT, c11 FLOAT, c12 BIGINT, c19 BINARY) " +
- "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE");
- executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE " +
- "default.allReadSupportedHiveDataTypes", testDataFile));
+ executeQuery(
+ "CREATE TABLE IF NOT EXISTS readtest (" +
+ " binary_field BINARY," +
+ " boolean_field BOOLEAN," +
+ " tinyint_field TINYINT," +
+ " decimal_field DECIMAL," +
+ " double_field DOUBLE," +
+ " float_field FLOAT," +
+ " int_field INT," +
+ " bigint_field BIGINT," +
+ " smallint_field SMALLINT," +
+ " string_field STRING," +
+ " varchar_field VARCHAR(50)," +
+ " timestamp_field TIMESTAMP," +
+ " date_field DATE" +
+ ") PARTITIONED BY (" +
+ " binary_part BINARY," +
+ " boolean_part BOOLEAN," +
+ " tinyint_part TINYINT," +
+ " decimal_part DECIMAL," +
+ " double_part DOUBLE," +
+ " float_part FLOAT," +
+ " int_part INT," +
+ " bigint_part BIGINT," +
+ " smallint_part SMALLINT," +
+ " string_part STRING," +
+ " varchar_part VARCHAR(50)," +
+ " timestamp_part TIMESTAMP," +
+ " date_part DATE" +
+ ") ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"
+ );
+
+ // Add a partition to table 'readtest'
+ executeQuery(
+ "ALTER TABLE readtest ADD IF NOT EXISTS PARTITION ( " +
+ " binary_part='binary', " +
+ " boolean_part='true', " +
+ " tinyint_part='64', " +
+ " decimal_part='3489423929323435243', " +
+ " double_part='8.345', " +
+ " float_part='4.67', " +
+ " int_part='123456', " +
+ " bigint_part='234235', " +
+ " smallint_part='3455', " +
+ " string_part='string', " +
+ " varchar_part='varchar', " +
+ " timestamp_part='2013-07-05 17:01:00', " +
+ " date_part='2013-07-05')"
+ );
+
+ // Load data into table 'readtest'
+ executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE default.readtest PARTITION (" +
+ " binary_part='binary', " +
+ " boolean_part='true', " +
+ " tinyint_part='64', " +
+ " decimal_part='3489423929323435243', " +
+ " double_part='8.345', " +
+ " float_part='4.67', " +
+ " int_part='123456', " +
+ " bigint_part='234235', " +
+ " smallint_part='3455', " +
+ " string_part='string', " +
+ " varchar_part='varchar', " +
+ " timestamp_part='2013-07-05 17:01:00', " +
+ " date_part='2013-07-05')", testDataFile));
// create a table that has all Hive types. This is to test how hive tables metadata is populated in
// Drill's INFORMATION_SCHEMA.
- executeQuery("CREATE TABLE IF NOT EXISTS allHiveDataTypes(" +
+ executeQuery("CREATE TABLE IF NOT EXISTS infoschematest(" +
"booleanType BOOLEAN, " +
"tinyintType TINYINT, " +
"smallintType SMALLINT, " +
@@ -176,9 +236,7 @@ public class HiveTestDataGenerator {
File file = getTempFile();
PrintWriter printWriter = new PrintWriter(file);
- printWriter.println("\\N,\\N,\\N,\\N,\\N,\\N,\\N,\\N,\\N");
- printWriter.println("-1,false,-1.1,,-1,-1,-1.0,-1,\\N");
- printWriter.println("1,true,1.1,1,1,1,1.0,1,YWJjZA==");
+ printWriter.println("YmluYXJ5ZmllbGQ=,false,34,3489423929323435243,8.345,4.67,123456,234235,3455,stringfield,varcharfield,2013-07-05 17:01:00,2013-07-05");
printWriter.close();
return file.getPath();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3bcead76/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
index e4ef332..1f70942 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveStorage.java
@@ -32,7 +32,6 @@ public class TestHiveStorage extends JdbcTestQueryBase{
public void testHiveReadWithDb() throws Exception{
testQuery("select * from hive.`default`.kv");
testQuery("select key from hive.`default`.kv group by key");
- testQuery("select * from hive.`default`.allreadsupportedhivedatatypes");
}
@Test
@@ -46,4 +45,55 @@ public class TestHiveStorage extends JdbcTestQueryBase{
testQuery("SELECT * FROM hive.`default`.empty_table");
}
+ @Test
+ public void testReadAllSupportedHiveDataTypes() throws Exception {
+ // There are known issues with displaying VarBinary in JDBC. So for now just execute the query and do not
+ // verify the results until display issues with VarBinary are resolved.
+ testQuery("SELECT * FROM hive.`default`.readtest");
+
+ /*
+ JdbcAssert.withFull("hive.default")
+ .sql("SELECT * FROM readtest")
+ .returns(
+ "binary_field=[B@7005f08f; " + // know issues with binary display
+ "boolean_field=false; " +
+ "tinyint_field=34; " +
+ "decimal_field=3489423929323435243; " +
+ "double_field=8.345; " +
+ "float_field=4.67; " +
+ "int_field=123456; " +
+ "bigint_field=234235; " +
+ "smallint_field=3455; " +
+ "string_field=stringfield; " +
+ "varchar_field=varcharfield; " +
+ "timestamp_field=2013-07-05T17:01:00.000-07:00; " +
+ "date_field=2013-07-05T00:00:00.000-07:00; " +
+ "binary_part=[B@7008383e; " + // know issues with binary display
+ "boolean_part=true; " +
+ "tinyint_part=64; " +
+ "decimal_part=3489423929323435243; " +
+ "double_part=8.345; " +
+ "float_part=4.67; " +
+ "int_part=123456; " +
+ "bigint_part=234235; " +
+ "smallint_part=3455; " +
+ "string_part=string; " +
+ "varchar_part=varchar; " +
+ "timestamp_part=2013-07-05T17:01:00.000-07:00; " +
+ "date_part=2013-07-05T00:00:00.000-07:00");
+ */
+ }
+
+ @Test
+ public void testOrderByOnHiveTable() throws Exception {
+ JdbcAssert.withFull("hive.default")
+ .sql("SELECT * FROM kv ORDER BY `value` DESC")
+ .returns(
+ "key=5; value= key_5\n" +
+ "key=4; value= key_4\n" +
+ "key=3; value= key_3\n" +
+ "key=2; value= key_2\n" +
+ "key=1; value= key_1\n"
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3bcead76/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 228bf84..e6e540a 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -58,10 +58,10 @@ public class TestMetadataDDL extends JdbcTestQueryBase {
JdbcAssert.withFull("hive.default")
.sql("SHOW TABLES")
.returns(
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=readtest\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
- "TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=infoschematest\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" +
- "TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"
);
@@ -82,10 +82,10 @@ public class TestMetadataDDL extends JdbcTestQueryBase {
JdbcAssert.withFull("dfs.tmp")
.sql("SHOW TABLES IN hive.`default`")
.returns(
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=readtest\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
- "TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=infoschematest\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" +
- "TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
}
@@ -247,7 +247,7 @@ public class TestMetadataDDL extends JdbcTestQueryBase {
JdbcAssert.withNoDefaultSchema()
.sql("SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION " +
"FROM INFORMATION_SCHEMA.`COLUMNS` " +
- "WHERE TABLE_SCHEMA = 'hive.default' AND TABLE_NAME = 'allhivedatatypes' AND " +
+ "WHERE TABLE_SCHEMA = 'hive.default' AND TABLE_NAME = 'infoschematest' AND " +
"(COLUMN_NAME = 'stringtype' OR COLUMN_NAME = 'varchartype' OR " +
"COLUMN_NAME = 'inttype' OR COLUMN_NAME = 'decimaltype')")
.returns(
[22/32] git commit: DRILL-932: Add support for bracketless complex
expressions in SQL queries.
Posted by ja...@apache.org.
DRILL-932: Add support for bracketless complex expressions in SQL queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e9ab7aa6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e9ab7aa6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e9ab7aa6
Branch: refs/heads/master
Commit: e9ab7aa6f303fccce22cfb83c44ce3edae526768
Parents: 78bc856
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Jun 10 20:24:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:16 2014 -0700
----------------------------------------------------------------------
.../drill/common/expression/FieldReference.java | 16 +-
.../drill/hbase/TestHBaseFilterPushDown.java | 4 +-
.../drill/hbase/TestHBaseProjectPushDown.java | 8 +-
exec/java-exec/src/main/codegen/data/Parser.tdd | 4 +
.../codegen/includes/compoundIdentifier.ftl | 45 ++++++
.../planner/common/DrillProjectRelBase.java | 8 +-
.../drill/exec/planner/logical/DrillOptiq.java | 2 +-
.../drill/exec/planner/sql/DrillSqlWorker.java | 4 +-
.../sql/parser/CompoundIdentifierConverter.java | 154 +++++++++++++++++++
.../sql/parser/DrillCompoundIdentifier.java | 144 +++++++++++++++++
.../exec/planner/sql/parser/SqlCreateTable.java | 16 +-
.../exec/planner/sql/parser/SqlCreateView.java | 25 ++-
.../planner/sql/parser/SqlDescribeTable.java | 12 +-
.../exec/planner/sql/parser/SqlDropView.java | 8 +-
.../exec/planner/sql/parser/SqlShowFiles.java | 10 +-
.../exec/planner/sql/parser/SqlShowSchemas.java | 10 +-
.../exec/planner/sql/parser/SqlShowTables.java | 12 +-
.../exec/planner/sql/parser/SqlUseSchema.java | 14 +-
.../exec/sql/TestSqlBracketlessSyntax.java | 68 ++++++++
.../apache/drill/exec/sql/TestWithClause.java | 42 +++++
.../complex/writer/TestComplexTypeReader.java | 36 ++---
21 files changed, 579 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
index bcc6b52..733e585 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -60,10 +60,22 @@ public class FieldReference extends SchemaPath {
checkSimpleString(value);
}
+ public static FieldReference getWithQuotedRef(CharSequence safeString){
+ return new FieldReference(safeString, ExpressionPosition.UNKNOWN, false);
+ }
+
+
public FieldReference(CharSequence value, ExpressionPosition pos) {
+ this(value, pos, true);
+ }
+
+ public FieldReference(CharSequence value, ExpressionPosition pos, boolean check) {
super(new NameSegment(value), pos);
- checkData();
- checkSimpleString(value);
+ if(check){
+ checkData();
+ checkSimpleString(value);
+ }
+
}
public FieldReference(String value, ExpressionPosition pos, MajorType dataType) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 2071e90..50dddeb 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -63,9 +63,9 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
- + " hbase.`[TABLE_NAME]` tableName\n"
+ + " hbase.`[TABLE_NAME]` t\n"
+ "WHERE\n"
- + " (row_key >= 'b5' OR row_key <= 'a2') AND (f['c1'] >= '1' OR f['c1'] is null)"
+ + " (row_key >= 'b5' OR row_key <= 'a2') AND (t.f.c1 >= '1' OR t.f.c1 is null)"
, 4);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
index 7037b9a..ce6f865 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
@@ -35,9 +35,9 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
public void testColumnWith1RowPushDown() throws Exception{
setColumnWidth(6);
runSQLVerifyCount("SELECT\n"
- + "f2['c7'] as `f[c7]`\n"
+ + "t.f2.c7 as `t.f2.c7`\n"
+ "FROM\n"
- + " hbase.`[TABLE_NAME]` tableName"
+ + " hbase.`[TABLE_NAME]` t"
, 1);
}
@@ -45,9 +45,9 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
public void testRowKeyAndColumnPushDown() throws Exception{
setColumnWidths(new int[] {8, 9, 6, 2, 6});
runSQLVerifyCount("SELECT\n"
- + "row_key, f['c1']*31 as `f[c1]*31`, f['c2'] as `f[c2]`, 5 as `5`, 'abc' as `'abc'`\n"
+ + "row_key, t.f.c1*31 as `t.f.c1*31`, t.f.c2 as `t.f.c2`, 5 as `5`, 'abc' as `'abc'`\n"
+ "FROM\n"
- + " hbase.`[TABLE_NAME]` tableName"
+ + " hbase.`[TABLE_NAME]` t"
, 6);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/codegen/data/Parser.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd
index d781f32..de7af2a 100644
--- a/exec/java-exec/src/main/codegen/data/Parser.tdd
+++ b/exec/java-exec/src/main/codegen/data/Parser.tdd
@@ -65,4 +65,8 @@
implementationFiles: [
"parserImpls.ftl"
]
+
+ includeCompoundIdentifier: false
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/codegen/includes/compoundIdentifier.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/compoundIdentifier.ftl b/exec/java-exec/src/main/codegen/includes/compoundIdentifier.ftl
new file mode 100644
index 0000000..50d8c20
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/includes/compoundIdentifier.ftl
@@ -0,0 +1,45 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+/**
+ * Parses a Drill compound identifier.
+ */
+SqlIdentifier CompoundIdentifier() :
+{
+ DrillCompoundIdentifier.Builder builder = DrillCompoundIdentifier.newBuilder();
+ String p;
+ int index;
+}
+{
+ p = Identifier()
+ {
+ builder.addString(p, getPos());
+ }
+ (
+ (
+ <DOT> p = Identifier()
+ {
+ builder.addString(p, getPos());
+ }
+ )
+ |
+ (
+ <LBRACKET>
+ index = UnsignedIntLiteral()
+ <RBRACKET>
+ {
+ builder.addIndex(index, getPos());
+ }
+ )
+ ) *
+ {
+ return builder.build();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillProjectRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillProjectRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillProjectRelBase.java
index 4629737..14817be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillProjectRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillProjectRelBase.java
@@ -55,14 +55,14 @@ public abstract class DrillProjectRelBase extends ProjectRelBase implements Dril
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner) {
if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
- return super.computeSelfCost(planner).multiplyBy(.1);
+ return super.computeSelfCost(planner).multiplyBy(.1);
}
-
+
// cost is proportional to the number of rows and number of columns being projected
double rowCount = RelMetadataQuery.getRowCount(this);
double cpuCost = DrillCostBase.PROJECT_CPU_COST * getRowType().getFieldCount();
DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
- return costFactory.makeCost(rowCount, cpuCost, 0, 0);
+ return costFactory.makeCost(rowCount, cpuCost, 0, 0);
}
private List<Pair<RexNode, String>> projects() {
@@ -73,7 +73,7 @@ public abstract class DrillProjectRelBase extends ProjectRelBase implements Dril
List<NamedExpression> expressions = Lists.newArrayList();
for (Pair<RexNode, String> pair : projects()) {
LogicalExpression expr = DrillOptiq.toDrill(context, getChild(), pair.left);
- expressions.add(new NamedExpression(expr, new FieldReference(pair.right)));
+ expressions.add(new NamedExpression(expr, FieldReference.getWithQuotedRef(pair.right)));
}
return expressions;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 3576622..dd30378 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -83,7 +83,7 @@ public class DrillOptiq {
public LogicalExpression visitInputRef(RexInputRef inputRef) {
final int index = inputRef.getIndex();
final RelDataTypeField field = input.getRowType().getFieldList().get(index);
- return new FieldReference(field.getName());
+ return FieldReference.getWithQuotedRef(field.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index eb2c891..c8d2548 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -38,6 +38,7 @@ import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.ExplainHandler;
import org.apache.drill.exec.planner.sql.handlers.SetOptionHandler;
+import org.apache.drill.exec.planner.sql.parser.CompoundIdentifierConverter;
import org.apache.drill.exec.planner.sql.parser.DrillSqlCall;
import org.apache.drill.exec.planner.sql.parser.impl.DrillParserImpl;
import org.apache.drill.exec.store.StoragePluginRegistry;
@@ -104,7 +105,8 @@ public class DrillSqlWorker {
}
public PhysicalPlan getPlan(String sql, Pointer<String> textPlan) throws SqlParseException, ValidationException, RelConversionException, IOException{
- SqlNode sqlNode = planner.parse(sql);
+ SqlNode originalNode = planner.parse(sql);
+ SqlNode sqlNode = originalNode.accept(new CompoundIdentifierConverter());
AbstractSqlHandler handler;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
new file mode 100644
index 0000000..9a44f9c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.parser;
+
+import java.util.List;
+import java.util.Map;
+
+import org.eigenbase.sql.SqlCall;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.SqlSelect;
+import org.eigenbase.sql.util.SqlShuttle;
+import org.eigenbase.sql.util.SqlVisitor;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+public class CompoundIdentifierConverter extends SqlShuttle {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompoundIdentifierConverter.class);
+
+ private boolean enableComplex = true;
+
+ @Override
+ public SqlNode visit(SqlIdentifier id) {
+ if(id instanceof DrillCompoundIdentifier){
+ if(enableComplex){
+ return ((DrillCompoundIdentifier) id).getAsSqlNode();
+ }else{
+ return ((DrillCompoundIdentifier) id).getAsCompoundIdentifier();
+ }
+
+ }else{
+ return id;
+ }
+ }
+
+ @Override
+ public SqlNode visit(final SqlCall call) {
+ // Handler creates a new copy of 'call' only if one or more operands
+ // change.
+ ArgHandler<SqlNode> argHandler = new ComplexExpressionAware(call);
+ call.getOperator().acceptCall(this, call, false, argHandler);
+ return argHandler.result();
+ }
+
+
+ private class ComplexExpressionAware implements ArgHandler<SqlNode> {
+ boolean update;
+ SqlNode[] clonedOperands;
+ RewriteType[] rewriteTypes;
+ private final SqlCall call;
+
+ public ComplexExpressionAware(SqlCall call) {
+ this.call = call;
+ this.update = false;
+ final List<SqlNode> operands = call.getOperandList();
+ this.clonedOperands = operands.toArray(new SqlNode[operands.size()]);
+ rewriteTypes = REWRITE_RULES.get(call.getClass());
+ }
+
+ public SqlNode result() {
+ if (update) {
+ return call.getOperator().createCall(
+ call.getFunctionQuantifier(),
+ call.getParserPosition(),
+ clonedOperands);
+ } else {
+ return call;
+ }
+ }
+
+ public SqlNode visitChild(
+ SqlVisitor<SqlNode> visitor,
+ SqlNode expr,
+ int i,
+ SqlNode operand) {
+ if (operand == null) {
+ return null;
+ }
+
+ boolean localEnableComplex = enableComplex;
+ if(rewriteTypes != null){
+ switch(rewriteTypes[i]){
+ case DISABLE:
+ enableComplex = false;
+ break;
+ case ENABLE:
+ enableComplex = true;
+ }
+ }
+ SqlNode newOperand = operand.accept(CompoundIdentifierConverter.this);
+ enableComplex = localEnableComplex;
+ if (newOperand != operand) {
+ update = true;
+ }
+ clonedOperands[i] = newOperand;
+ return newOperand;
+ }
+ }
+
+ static final Map<Class<? extends SqlCall>, RewriteType[]> REWRITE_RULES;
+
+ enum RewriteType {
+ UNCHANGED, DISABLE, ENABLE;
+ }
+
+ static {
+ final RewriteType E =RewriteType.ENABLE;
+ final RewriteType D =RewriteType.DISABLE;
+ final RewriteType U =RewriteType.UNCHANGED;
+
+ Map<Class<? extends SqlCall>, RewriteType[]> rules = Maps.newHashMap();
+
+ //SqlNodeList keywordList,
+ //SqlNodeList selectList,
+ //SqlNode fromClause,
+ //SqlNode whereClause,
+ //SqlNodeList groupBy,
+ //SqlNode having,
+ //SqlNodeList windowDecls,
+ //SqlNodeList orderBy,
+ //SqlNode offset,
+ //SqlNode fetch,
+ rules.put(SqlSelect.class, R(D, E, D, E, E, E, D, E, D, D));
+ rules.put(SqlCreateTable.class, R(D, D, E));
+ rules.put(SqlCreateView.class, R(D, E, E, D));
+ rules.put(SqlDescribeTable.class, R(D, D, E));
+ rules.put(SqlDropView.class, R(D));
+ rules.put(SqlShowFiles.class, R(D));
+ rules.put(SqlShowSchemas.class, R(D, D));
+ rules.put(SqlUseSchema.class, R(D));
+ REWRITE_RULES = ImmutableMap.copyOf(rules);
+ }
+
+ private static RewriteType[] R(RewriteType... types){
+ return types;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/DrillCompoundIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/DrillCompoundIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/DrillCompoundIdentifier.java
new file mode 100644
index 0000000..d718efa
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/DrillCompoundIdentifier.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.parser;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.eigenbase.sql.SqlBasicCall;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlLiteral;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.sql.parser.SqlParserPos;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
+
+public class DrillCompoundIdentifier extends SqlIdentifier{
+
+ List<IdentifierHolder> ids;
+
+ private static List<String> getNames(List<IdentifierHolder> identifiers){
+ List<String> names = Lists.newArrayListWithCapacity(identifiers.size());
+ for(IdentifierHolder h : identifiers){
+ names.add(h.value);
+ }
+ return names;
+ }
+
+ public DrillCompoundIdentifier(List<IdentifierHolder> identifiers) {
+ super(getNames(identifiers), identifiers.get(0).parserPos);
+ this.ids = identifiers;
+ }
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillCompoundIdentifier.class);
+
+ public static Builder newBuilder(){
+ return new Builder();
+ }
+
+ public static class Builder {
+ private List<IdentifierHolder> identifiers = Lists.newArrayList();
+
+ public DrillCompoundIdentifier build(){
+ return new DrillCompoundIdentifier(identifiers);
+ }
+
+ public void addString(String name, SqlParserPos pos){
+ identifiers.add(new IdentifierHolder(name, pos, false));
+ }
+
+ public void addIndex(int index, SqlParserPos pos){
+ identifiers.add(new IdentifierHolder(Integer.toString(index), pos, true));
+ }
+ }
+
+ public SqlNode getAsSqlNode(){
+ if(ids.size() == 1){
+ return new SqlIdentifier(Collections.singletonList(ids.get(0).value), ids.get(0).parserPos);
+ }
+
+ int startIndex;
+ SqlNode node;
+
+ if(ids.get(1).isArray()){
+ // handle everything post zero index as item operator.
+ startIndex = 1;
+ node = new SqlIdentifier( //
+ ImmutableList.of(ids.get(0).value), //
+ null, //
+ ids.get(0).parserPos, //
+ ImmutableList.of(ids.get(0).parserPos));
+ }else{
+ // handle everything post two index as item operator.
+ startIndex = 2;
+ node = new SqlIdentifier( //
+ ImmutableList.of(ids.get(0).value, ids.get(1).value), //
+ null, //
+ ids.get(0).parserPos, //
+ ImmutableList.of(ids.get(0).parserPos, ids.get(1).parserPos));
+
+ }
+ for(int i = startIndex ; i < ids.size(); i++){
+ node = ids.get(i).getNode(node);
+ }
+
+ return node;
+ }
+
+
+ public SqlNode getAsCompoundIdentifier(){
+ List<String> names = Lists.newArrayListWithCapacity(ids.size());
+ List<SqlParserPos> pos = Lists.newArrayListWithCapacity(ids.size());
+ for(int i =0; i < ids.size(); i++){
+ IdentifierHolder holder = ids.get(i);
+ names.add(holder.value);
+ pos.add(holder.parserPos);
+ }
+ return new SqlIdentifier(names, null, pos.get(0), pos);
+ }
+
+ private static class IdentifierHolder{
+ String value;
+ SqlParserPos parserPos;
+ boolean isArray;
+
+ public IdentifierHolder(String value, SqlParserPos parserPos, boolean isArray) {
+ super();
+ this.isArray = isArray;
+ this.value = value;
+ this.parserPos = parserPos;
+ }
+
+ public boolean isArray(){
+ return isArray;
+ }
+
+ public SqlNode getNode(SqlNode node){
+ SqlLiteral literal;
+ if(isArray){
+ literal = SqlLiteral.createExactNumeric(value, parserPos);
+ }else{
+ literal = SqlLiteral.createCharString(value, parserPos);
+ }
+ return new SqlBasicCall(SqlStdOperatorTable.ITEM, new SqlNode[]{ node, literal }, parserPos);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
index 1c916a7..0f0cf0a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateTable.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import java.util.List;
+import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
import net.hydromatic.optiq.tools.Planner;
@@ -37,7 +38,11 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
public class SqlCreateTable extends DrillSqlCall {
- public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_TABLE", SqlKind.OTHER);
+ public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_TABLE", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlCreateTable(pos, (SqlIdentifier) operands[0], (SqlNodeList) operands[1], operands[2]);
+ }
+ };
private SqlIdentifier tblName;
private SqlNodeList fieldList;
@@ -57,8 +62,11 @@ public class SqlCreateTable extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
- if (fieldList == null) return ImmutableList.of(tblName, fieldList);
- else return ImmutableList.of(tblName, query, fieldList);
+ List<SqlNode> ops = Lists.newArrayList();
+ ops.add(tblName);
+ ops.add(fieldList);
+ ops.add(query);
+ return ops;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
index f3c8bde..6f798b8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlCreateView.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import java.util.List;
+import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
import net.hydromatic.optiq.tools.Planner;
@@ -34,11 +35,14 @@ import org.eigenbase.sql.SqlSpecialOperator;
import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import java.util.List;
public class SqlCreateView extends DrillSqlCall {
- public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_VIEW", SqlKind.OTHER);
+ public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("CREATE_VIEW", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlCreateView(pos, (SqlIdentifier) operands[0], (SqlNodeList) operands[1], operands[2], (SqlLiteral) operands[3]);
+ }
+ };
private SqlIdentifier viewName;
private SqlNodeList fieldList;
@@ -46,6 +50,11 @@ public class SqlCreateView extends DrillSqlCall {
private boolean replaceView;
public SqlCreateView(SqlParserPos pos, SqlIdentifier viewName, SqlNodeList fieldList,
+ SqlNode query, SqlLiteral replaceView) {
+ this(pos, viewName, fieldList, query, replaceView.booleanValue());
+ }
+
+ public SqlCreateView(SqlParserPos pos, SqlIdentifier viewName, SqlNodeList fieldList,
SqlNode query, boolean replaceView) {
super(pos);
this.viewName = viewName;
@@ -61,8 +70,12 @@ public class SqlCreateView extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
- return ImmutableList.of(viewName, fieldList, query,
- SqlLiteral.createBoolean(replaceView, SqlParserPos.ZERO));
+ List<SqlNode> ops = Lists.newArrayList();
+ ops.add(viewName);
+ ops.add(fieldList);
+ ops.add(query);
+ ops.add(SqlLiteral.createBoolean(replaceView, SqlParserPos.ZERO));
+ return ops;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
index be804a9..8b022b0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDescribeTable.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.planner.sql.parser;
-import java.util.List;
+import com.google.common.collect.Lists;
import net.hydromatic.optiq.tools.Planner;
@@ -45,7 +45,11 @@ public class SqlDescribeTable extends DrillSqlCall {
private final SqlNode columnQualifier;
public static final SqlSpecialOperator OPERATOR =
- new SqlSpecialOperator("DESCRIBE_TABLE", SqlKind.OTHER);
+ new SqlSpecialOperator("DESCRIBE_TABLE", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlDescribeTable(pos, (SqlIdentifier) operands[0], (SqlIdentifier) operands[1], operands[2]);
+ }
+ };
public SqlDescribeTable(SqlParserPos pos, SqlIdentifier table, SqlIdentifier column, SqlNode columnQualifier) {
super(pos);
@@ -63,8 +67,8 @@ public class SqlDescribeTable extends DrillSqlCall {
public List<SqlNode> getOperandList() {
List<SqlNode> opList = Lists.newArrayList();
opList.add(table);
- if (column != null) opList.add(column);
- if (columnQualifier != null) opList.add(columnQualifier);
+ opList.add(column);
+ opList.add(columnQualifier);
return opList;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
index 20a7952..b86bd20 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropView.java
@@ -35,7 +35,11 @@ import org.eigenbase.sql.parser.SqlParserPos;
import com.google.common.collect.ImmutableList;
public class SqlDropView extends DrillSqlCall {
- public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DROP_VIEW", SqlKind.OTHER);
+ public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DROP_VIEW", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlDropView(pos, (SqlIdentifier) operands[0]);
+ }
+ };
private SqlIdentifier viewName;
@@ -51,7 +55,7 @@ public class SqlDropView extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
- return ImmutableList.of((SqlNode)viewName);
+ return Collections.singletonList((SqlNode)viewName);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
index 7e967b6..c929db6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowFiles.java
@@ -43,7 +43,11 @@ public class SqlShowFiles extends DrillSqlCall {
private final SqlIdentifier db;
public static final SqlSpecialOperator OPERATOR =
- new SqlSpecialOperator("SHOW_FILES", SqlKind.OTHER);
+ new SqlSpecialOperator("SHOW_FILES", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlShowFiles(pos, (SqlIdentifier) operands[0]);
+ }
+ };
public SqlShowFiles(SqlParserPos pos, SqlIdentifier db) {
super(pos);
@@ -57,9 +61,7 @@ public class SqlShowFiles extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
- List<SqlNode> opList = Lists.newArrayList();
- if (db != null) opList.add(db);
- return opList;
+ return Collections.singletonList( (SqlNode) db);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
index 2ef6553..03e4e1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowSchemas.java
@@ -43,7 +43,11 @@ public class SqlShowSchemas extends DrillSqlCall {
private final SqlNode whereClause;
public static final SqlSpecialOperator OPERATOR =
- new SqlSpecialOperator("SHOW_SCHEMAS", SqlKind.OTHER);
+ new SqlSpecialOperator("SHOW_SCHEMAS", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlShowSchemas(pos, operands[0], operands[1]);
+ }
+ };
public SqlShowSchemas(SqlParserPos pos, SqlNode likePattern, SqlNode whereClause) {
super(pos);
@@ -59,8 +63,8 @@ public class SqlShowSchemas extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
List<SqlNode> opList = Lists.newArrayList();
- if (likePattern != null) opList.add(likePattern);
- if (whereClause != null) opList.add(whereClause);
+ opList.add(likePattern);
+ opList.add(whereClause);
return opList;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
index 93de127..ae95944 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlShowTables.java
@@ -45,7 +45,11 @@ public class SqlShowTables extends DrillSqlCall {
private final SqlNode whereClause;
public static final SqlSpecialOperator OPERATOR =
- new SqlSpecialOperator("SHOW_TABLES", SqlKind.OTHER);
+ new SqlSpecialOperator("SHOW_TABLES", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlShowTables(pos, (SqlIdentifier) operands[0], operands[1], operands[2]);
+ }
+ };
public SqlShowTables(SqlParserPos pos, SqlIdentifier db, SqlNode likePattern, SqlNode whereClause) {
super(pos);
@@ -62,9 +66,9 @@ public class SqlShowTables extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
List<SqlNode> opList = Lists.newArrayList();
- if (db != null) opList.add(db);
- if (likePattern != null) opList.add(likePattern);
- if (whereClause != null) opList.add(whereClause);
+ opList.add(db);
+ opList.add(likePattern);
+ opList.add(whereClause);
return opList;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
index 554bfbf..76c7df3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlUseSchema.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.sql.parser;
+import java.util.Collections;
import java.util.List;
import net.hydromatic.optiq.tools.Planner;
@@ -24,23 +25,28 @@ import net.hydromatic.optiq.tools.Planner;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
import org.apache.drill.exec.planner.sql.handlers.UseSchemaHandler;
+import org.eigenbase.sql.SqlCall;
import org.eigenbase.sql.SqlIdentifier;
import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.SqlSpecialOperator;
import org.eigenbase.sql.SqlWriter;
import org.eigenbase.sql.parser.SqlParserPos;
-import com.google.common.collect.ImmutableList;
-
/**
* Sql parser tree node to represent <code>USE SCHEMA</code> statement.
*/
public class SqlUseSchema extends DrillSqlCall {
public static final SqlSpecialOperator OPERATOR =
- new SqlSpecialOperator("USE_SCHEMA", SqlKind.OTHER);
+ new SqlSpecialOperator("USE_SCHEMA", SqlKind.OTHER){
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlUseSchema(pos, (SqlIdentifier) operands[0]);
+ }
+ };
+
private SqlIdentifier schema;
public SqlUseSchema(SqlParserPos pos, SqlIdentifier schema) {
@@ -51,7 +57,7 @@ public class SqlUseSchema extends DrillSqlCall {
@Override
public List<SqlNode> getOperandList() {
- return ImmutableList.of((SqlNode)schema);
+ return Collections.singletonList((SqlNode)schema);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
new file mode 100644
index 0000000..1204e7c
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSqlBracketlessSyntax.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.sql;
+
+import net.hydromatic.optiq.config.Lex;
+import net.hydromatic.optiq.tools.Frameworks;
+import net.hydromatic.optiq.tools.Planner;
+import net.hydromatic.optiq.tools.StdFrameworkConfig;
+
+import org.apache.drill.exec.planner.sql.DrillConvertletTable;
+import org.apache.drill.exec.planner.sql.parser.CompoundIdentifierConverter;
+import org.apache.drill.exec.planner.sql.parser.impl.DrillParserImpl;
+import org.eigenbase.sql.SqlNode;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestSqlBracketlessSyntax {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSqlBracketlessSyntax.class);
+
+ @Test
+ public void checkComplexExpressionParsing() throws Exception{
+ StdFrameworkConfig config = StdFrameworkConfig.newBuilder() //
+ .lex(Lex.MYSQL) //
+ .parserFactory(DrillParserImpl.FACTORY) //
+ .defaultSchema(Frameworks.createRootSchema(false)) //
+ .convertletTable(new DrillConvertletTable()) //
+ .build();
+ Planner planner = Frameworks.getPlanner(config);
+
+ SqlNode node = planner.parse(""
+ + "select a[4].c \n"
+ + "from x.y.z \n"
+ + "where a.c.b = 5 and x[2] = 7 \n"
+ + "group by d \n"
+ + "having a.c < 5 \n"
+ + "order by x.a.a.a.a.a");
+
+ String expected = "SELECT `a`[4]['c']\n" +
+ "FROM `x`.`y`.`z`\n" +
+ "WHERE `a`.`c`['b'] = 5 AND `x`[2] = 7\n" +
+ "GROUP BY `d`\n" +
+ "HAVING `a`.`c` < 5\n" +
+ "ORDER BY `x`.`a`['a']['a']['a']['a']";
+
+
+ SqlNode rewritten = node.accept(new CompoundIdentifierConverter());
+ String rewrittenQuery = rewritten.toString();
+
+ Assert.assertEquals(expected, rewrittenQuery);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
new file mode 100644
index 0000000..3a32045
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.sql;
+
+import org.apache.drill.BaseTestQuery;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestWithClause extends BaseTestQuery {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestWithClause.class);
+
+ @Test
+ public void withClause() throws Exception {
+ test("with alpha as (select * from sys.options where type = 'SYSTEM') \n" +
+ "\n" +
+ "select * from alpha");
+ }
+
+ @Test
+ @Ignore
+ public void withClauseWithAliases() throws Exception {
+ test("with alpha (x,y) as (select name, kind from sys.options where type = 'SYSTEM') \n" +
+ "\n" +
+ "select x, y from alpha");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e9ab7aa6/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 74c724d..3da3543 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -25,21 +25,21 @@ public class TestComplexTypeReader extends BaseTestQuery{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class);
@Test
- // Repeated map (map) -> json.
+ // Repeated map (map) -> json.
public void testX() throws Exception{
test("select convert_to(z[0], 'JSON') from cp.`jsoninput/input2.json`;");
}
@Test
- //map -> json.
+ //map -> json.
public void testX2() throws Exception{
test("select convert_to(x, 'JSON') from cp.`jsoninput/input2.json`;");
}
@Test
- //Map (mapfield) -> json.
+ //Map (mapfield) -> json.
public void testX3() throws Exception{
- test("select convert_to(x['y'], 'JSON') from cp.`jsoninput/input2.json`;");
+ test("select convert_to(tbl.x.y, 'JSON') from cp.`jsoninput/input2.json` tbl;");
}
@Test
@@ -53,9 +53,9 @@ public class TestComplexTypeReader extends BaseTestQuery{
public void testX5() throws Exception{
test("select convert_to(`integer`, 'JSON') from cp.`jsoninput/input2.json`;");
}
-
+
@Test
- // repeated map -> json.
+ // repeated map -> json.
public void testX6() throws Exception{
test("select convert_to(z, 'JSON') from cp.`jsoninput/input2.json`;");
}
@@ -90,7 +90,7 @@ public class TestComplexTypeReader extends BaseTestQuery{
@Test
public void testY3() throws Exception{
- test("select x['y'] from cp.`jsoninput/input2.json`;");
+ test("select tbl.x.y from cp.`jsoninput/input2.json` tbl;");
}
@Test
@@ -99,13 +99,13 @@ public class TestComplexTypeReader extends BaseTestQuery{
}
@Test
- //repeated list (Repeated BigInt)
+ //repeated list (Repeated BigInt)
public void testZ() throws Exception{
test("select rl[1] from cp.`jsoninput/input2.json`;");
}
@Test
- //repeated list (Repeated BigInt ( BigInt) ) )
+ //repeated list (Repeated BigInt ( BigInt) ) )
public void testZ1() throws Exception{
test("select rl[0][1] from cp.`jsoninput/input2.json`;");
}
@@ -123,27 +123,27 @@ public class TestComplexTypeReader extends BaseTestQuery{
}
@Test
- //repeated map --> Json. It will go beyond the buffer of size 256 allocated in setup.
+ //repeated map --> Json. It will go beyond the buffer of size 256 allocated in setup.
public void testA0() throws Exception{
test(" select convert_to(types, 'JSON') from cp.`jsoninput/vvtypes.json`;");
}
@Test
- //repeated map (map) --> Json.
+ //repeated map (map) --> Json.
public void testA1() throws Exception{
test(" select convert_to(types[1], 'JSON') from cp.`jsoninput/vvtypes.json`;");
}
@Test
- //repeated map (map (repeated map) ) --> Json.
+ //repeated map (map (repeated map) ) --> Json.
public void testA2() throws Exception{
- test(" select convert_to(types[1]['minor'], 'JSON') from cp.`jsoninput/vvtypes.json`;");
+ test(" select convert_to(types[1].minor, 'JSON') from cp.`jsoninput/vvtypes.json`;");
}
-
+
@Test
- //repeated map (map( repeated map (map (varchar)))) --> Json.
+ //repeated map (map( repeated map (map (varchar)))) --> Json.
public void testA3() throws Exception{
- test(" select convert_to(types[1]['minor'][0]['valueHolder'], 'JSON') from cp.`jsoninput/vvtypes.json`;");
+ test(" select convert_to(types[1].minor[0].valueHolder, 'JSON') from cp.`jsoninput/vvtypes.json`;");
}
@Test
@@ -155,13 +155,13 @@ public class TestComplexTypeReader extends BaseTestQuery{
@Test
//repeated map (map (repeated map) ).
public void testB2() throws Exception{
- test(" select types[1]['minor'] from cp.`jsoninput/vvtypes.json`;");
+ test(" select types[1].minor from cp.`jsoninput/vvtypes.json`;");
}
@Test
//repeated map (map( repeated map (map (varchar)))).
public void testB3() throws Exception{
- test(" select types[1]['minor'][0]['valueHolder'] from cp.`jsoninput/vvtypes.json`;");
+ test(" select types[1].minor[0].valueholder from cp.`jsoninput/vvtypes.json`;");
}
}
[24/32] DRILL-933: Remove old physical operator cost & size concepts,
add automatic size-based parallelization
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
index c295ac3..a638741 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionExchange.java
@@ -21,12 +21,9 @@ import java.util.List;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.physical.base.AbstractExchange;
-import org.apache.drill.exec.physical.base.ExchangeCost;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.Receiver;
import org.apache.drill.exec.physical.base.Sender;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -39,11 +36,11 @@ public class UnionExchange extends AbstractExchange{
private List<DrillbitEndpoint> senderLocations;
private DrillbitEndpoint destinationLocation;
-
+
public UnionExchange(@JsonProperty("child") PhysicalOperator child) {
super(child);
}
-
+
@Override
public void setupSenders(List<DrillbitEndpoint> senderLocations) {
this.senderLocations = senderLocations;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ExecPlanner.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ExecPlanner.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ExecPlanner.java
deleted file mode 100644
index 2f4f7e2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/ExecPlanner.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.physical.PhysicalPlan;
-import org.apache.drill.exec.work.QueryWorkUnit;
-
-
-/**
- * Decides level of paralellization.
- * Generates smaller physical plans
- */
-public interface ExecPlanner {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExecPlanner.class);
-
- public QueryWorkUnit getWorkUnit(QueryContext context, PhysicalPlan plan, int maxWidth) throws ExecutionSetupException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/SimpleExecPlanner.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/SimpleExecPlanner.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/SimpleExecPlanner.java
deleted file mode 100644
index 132fde7..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/SimpleExecPlanner.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.physical.PhysicalPlan;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.planner.fragment.Fragment;
-import org.apache.drill.exec.planner.fragment.MakeFragmentsVisitor;
-import org.apache.drill.exec.planner.fragment.PlanningSet;
-import org.apache.drill.exec.planner.fragment.SimpleParallelizer;
-import org.apache.drill.exec.planner.fragment.StatsCollector;
-import org.apache.drill.exec.server.options.OptionList;
-import org.apache.drill.exec.work.QueryWorkUnit;
-
-/**
- * Parallelization is based on available nodes with source or target data. Nodes that are "overloaded" are excluded from execution.
- */
-public class SimpleExecPlanner implements ExecPlanner{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleExecPlanner.class);
-
- private MakeFragmentsVisitor fragmenter = new MakeFragmentsVisitor();
- private SimpleParallelizer parallelizer = new SimpleParallelizer();
-
- @Override
- public QueryWorkUnit getWorkUnit(QueryContext context, PhysicalPlan plan, int maxWidth) throws ExecutionSetupException {
-
- // get the root physical operator and split the plan into sub fragments.
- PhysicalOperator root = plan.getSortedOperators(false).iterator().next();
- Fragment fragmentRoot = root.accept(fragmenter, null);
-
- // generate a planning set and collect stats.
- PlanningSet planningSet = StatsCollector.collectStats(fragmentRoot);
-
- int maxWidthPerEndpoint = context.getConfig().getInt(ExecConstants.MAX_WIDTH_PER_ENDPOINT);
-
- parallelizer.setGlobalMaxWidth(maxWidth).setMaxWidthPerEndpoint(maxWidthPerEndpoint);
- return parallelizer.getFragments(new OptionList(), context.getCurrentEndpoint(),
- context.getQueryId(), context.getActiveEndpoints(), context.getPlanReader(), fragmentRoot, planningSet);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MaterializedFragment.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MaterializedFragment.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MaterializedFragment.java
deleted file mode 100644
index 2624940..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MaterializedFragment.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.fragment;
-
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
-public class MaterializedFragment {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaterializedFragment.class);
-
- final PhysicalOperator root;
- final DrillbitEndpoint endpoint;
- final long queryId;
- final int majorFragmentId;
- final int minorFragmentId;
- final OperatorCost cost;
-
- public MaterializedFragment(PhysicalOperator root, DrillbitEndpoint endpoint, long queryId, int majorFragmentId,
- int minorFragmentId, OperatorCost cost) {
- super();
- this.root = root;
- this.endpoint = endpoint;
- this.queryId = queryId;
- this.majorFragmentId = majorFragmentId;
- this.minorFragmentId = minorFragmentId;
- this.cost = cost;
- }
-
- public PhysicalOperator getRoot() {
- return root;
- }
-
- public DrillbitEndpoint getEndpoint() {
- return endpoint;
- }
-
- public long getQueryId() {
- return queryId;
- }
-
- public int getMajorFragmentId() {
- return majorFragmentId;
- }
-
- public int getMinorFragmentId() {
- return minorFragmentId;
- }
-
- public OperatorCost getCost() {
- return cost;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index 9b84724..053580b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -23,8 +23,10 @@ import java.util.List;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
+import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.base.FragmentRoot;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.planner.PhysicalPlanReader;
@@ -47,39 +49,31 @@ import com.google.common.collect.Lists;
* is done based on round robin assignment ordered by operator affinity (locality) to available execution Drillbits.
*/
public class SimpleParallelizer {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleParallelizer.class);
- private final Materializer materializer = new Materializer();
- /**
- * The maximum level or parallelization any stage of the query can do. Note that while this
- * might be the number of active Drillbits, realistically, this could be well beyond that
- * number of we want to do things like speed results return.
- */
- private int globalMaxWidth;
- public SimpleParallelizer setGlobalMaxWidth(int globalMaxWidth) {
- this.globalMaxWidth = globalMaxWidth;
- return this;
- }
- /**
- * Limits the maximum level of parallelization to this factor time the number of Drillbits
- */
- private int maxWidthPerEndpoint;
- public SimpleParallelizer setMaxWidthPerEndpoint(int maxWidthPerEndpoint) {
- this.maxWidthPerEndpoint = maxWidthPerEndpoint;
- return this;
- }
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleParallelizer.class);
+ private final Materializer materializer = new Materializer();
+ private final long parallelizationThreshold;
+ private final int maxWidthPerNode;
+ private final int maxGlobalWidth;
+ private double affinityFactor;
+
+ public SimpleParallelizer(QueryContext context){
+ this.parallelizationThreshold = context.getOptions().getOption(ExecConstants.SLICE_TARGET).num_val;
+ this.maxWidthPerNode = context.getOptions().getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val.intValue();
+ this.maxGlobalWidth = context.getOptions().getOption(ExecConstants.MAX_WIDTH_GLOBAL_KEY).num_val.intValue();
+ this.affinityFactor = context.getOptions().getOption(ExecConstants.AFFINITY_FACTOR_KEY).float_val.intValue();
+ }
- /**
- * Factor by which a node with endpoint affinity will be favored while creating assignment
- */
- private double affinityFactor = 1.2f;
- public SimpleParallelizer setAffinityFactor(double affinityFactor) {
+ public SimpleParallelizer(long parallelizationThreshold, int maxWidthPerNode, int maxGlobalWidth, double affinityFactor){
+ this.parallelizationThreshold = parallelizationThreshold;
+ this.maxWidthPerNode = maxWidthPerNode;
+ this.maxGlobalWidth = maxGlobalWidth;
this.affinityFactor = affinityFactor;
- return this;
}
+
/**
* Generate a set of assigned fragments based on the provided planningSet. Do not allow parallelization stages to go
* beyond the global max width.
@@ -151,11 +145,7 @@ public class SimpleParallelizer {
.setQueryId(queryId) //
.build();
PlanFragment fragment = PlanFragment.newBuilder() //
- .setCpuCost(stats.getCpuCost()) //
- .setDiskCost(stats.getDiskCost()) //
.setForeman(foremanNode) //
- .setMemoryCost(stats.getMemoryCost()) //
- .setNetworkCost(stats.getNetworkCost()) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
@@ -182,25 +172,19 @@ public class SimpleParallelizer {
}
private void assignEndpoints(Collection<DrillbitEndpoint> allNodes, PlanningSet planningSet) throws PhysicalOperatorSetupException {
- // First we determine the amount of parallelization for a fragment. This will be between 1 and maxWidth based on
- // cost. (Later could also be based on cluster operation.) then we decide endpoints based on affinity (later this
- // could be based on endpoint load)
+ // for each node, set the width based on the parallelization threshold and cluster width.
for (Wrapper wrapper : planningSet) {
Stats stats = wrapper.getStats();
+ double targetSlices = stats.getTotalCost()/parallelizationThreshold;
+ int targetIntSlices = (int) Math.ceil(targetSlices);
+
// figure out width.
- int width = Math.min(stats.getMaxWidth(), globalMaxWidth);
- float diskCost = stats.getDiskCost();
-// logger.debug("Frag max width: {} and diskCost: {}", stats.getMaxWidth(), diskCost);
-
- // TODO: right now we'll just assume that each task is cost 1 so we'll set the breadth at the lesser of the number
- // of tasks or the maximum width of the fragment.
- if (diskCost < width) {
-// width = (int) diskCost;
- }
+ int width = Math.min(targetIntSlices, Math.min(stats.getMaxWidth(), maxGlobalWidth));
+
- width = Math.min(width, maxWidthPerEndpoint*allNodes.size());
+ width = Math.min(width, maxWidthPerNode*allNodes.size());
if (width < 1) width = 1;
// logger.debug("Setting width {} on fragment {}", width, wrapper);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
index 540f3f3..eda364b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
@@ -17,54 +17,33 @@
*/
package org.apache.drill.exec.planner.fragment;
-import org.apache.drill.exec.physical.OperatorCost;
public class Stats {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Stats.class);
-
+
private int maxWidth = Integer.MAX_VALUE;
- private float networkCost;
- private float diskCost;
- private float memoryCost;
- private float cpuCost;
-
+ private double totalCost;
+
public void addMaxWidth(int maxWidth){
this.maxWidth = Math.min(this.maxWidth, maxWidth);
}
-
- public void addCost(OperatorCost cost){
- networkCost += cost.getNetwork();
- diskCost += cost.getDisk();
- memoryCost += cost.getMemory();
- cpuCost += cost.getCpu();
+
+ public void addCost(double cost){
+ totalCost += cost;
}
public int getMaxWidth() {
return maxWidth;
}
- public float getNetworkCost() {
- return networkCost;
- }
-
- public float getDiskCost() {
- return diskCost;
+ @Override
+ public String toString() {
+ return "Stats [maxWidth=" + maxWidth + ", totalCost=" + totalCost + "]";
}
- public float getMemoryCost() {
- return memoryCost;
+ public double getTotalCost(){
+ return totalCost;
}
- public float getCpuCost() {
- return cpuCost;
- }
- @Override
- public String toString() {
- return "FragmentStats [maxWidth=" + maxWidth + ", networkCost=" + networkCost + ", diskCost=" + diskCost
- + ", memoryCost=" + memoryCost + ", cpuCost=" + cpuCost + "]";
- }
-
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
index 096ac08..3fc3b89 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
@@ -64,21 +64,19 @@ public class StatsCollector {
@Override
public Void visitSendingExchange(Exchange exchange, Wrapper wrapper) throws RuntimeException {
Stats stats = wrapper.getStats();
- stats.addCost(exchange.getAggregateSendCost());
stats.addMaxWidth(exchange.getMaxSendWidth());
return super.visitSendingExchange(exchange, wrapper);
}
@Override
public Void visitReceivingExchange(Exchange exchange, Wrapper wrapper) throws RuntimeException {
- wrapper.getStats().addCost(exchange.getAggregateReceiveCost());
// no traversal since it would cross fragment boundary.
return null;
}
@Override
public Void visitGroupScan(GroupScan groupScan, Wrapper wrapper) {
- Stats stats = wrapper.getStats();
+ Stats stats = wrapper.getStats();
stats.addMaxWidth(groupScan.getMaxParallelizationWidth());
return super.visitGroupScan(groupScan, wrapper);
}
@@ -86,7 +84,7 @@ public class StatsCollector {
@Override
public Void visitSubScan(SubScan subScan, Wrapper wrapper) throws RuntimeException {
// TODO - implement this
- return super.visitOp(subScan, wrapper);
+ return visitOp(subScan, wrapper);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
index 586b0ec..dcbfb3d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
@@ -27,10 +27,9 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.LogicalOperator;
import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.planner.common.DrillScanRelBase;
import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
import org.apache.drill.exec.planner.physical.PrelUtil;
@@ -111,7 +110,7 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
@Override
public double getRows() {
- return this.groupScan.getSize().getRecordCount();
+ return this.groupScan.getScanStats().getRecordCount();
}
/// TODO: this method is same as the one for ScanPrel...eventually we should consolidate
@@ -119,16 +118,15 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
/// by both logical and physical rels.
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner) {
- Size scanSize = this.groupScan.getSize();
+ ScanStats stats = this.groupScan.getScanStats();
int columnCount = this.getRowType().getFieldCount();
if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
- OperatorCost scanCost = this.groupScan.getCost();
- return planner.getCostFactory().makeCost(scanSize.getRecordCount() * columnCount, scanCost.getCpu(), scanCost.getDisk());
+ return planner.getCostFactory().makeCost(stats.getRecordCount() * columnCount, stats.getCpuCost(), stats.getDiskCost());
}
// double rowCount = RelMetadataQuery.getRowCount(this);
- double rowCount = scanSize.getRecordCount();
+ double rowCount = stats.getRecordCount();
double cpuCost = rowCount * columnCount; // for now, assume cpu cost is proportional to row count.
// Even though scan is reading from disk, in the currently generated plans all plans will
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPruleBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPruleBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPruleBase.java
index 4edeaf8..1b1cc94 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPruleBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPruleBase.java
@@ -25,7 +25,7 @@ import net.hydromatic.optiq.util.BitSets;
import org.apache.drill.exec.planner.logical.DrillAggregateRel;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
import org.eigenbase.rel.AggregateCall;
-import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.RelOptRuleCall;
import org.eigenbase.relopt.RelOptRuleOperand;
@@ -61,7 +61,9 @@ public abstract class AggPruleBase extends Prule {
// currently won't generate a 2 phase plan.
protected boolean create2PhasePlan(RelOptRuleCall call, DrillAggregateRel aggregate) {
PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
- if (! settings.isMultiPhaseAggEnabled() || settings.isSingleMode()) {
+ RelNode child = call.rel(0).getInputs().get(0);
+ boolean smallInput = child.computeSelfCost(call.getPlanner()).getRows() < settings.getSliceTarget();
+ if (! settings.isMultiPhaseAggEnabled() || settings.isSingleMode() || smallInput) {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/BroadcastExchangePrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/BroadcastExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/BroadcastExchangePrel.java
index 8b1c720..39ab26b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/BroadcastExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/BroadcastExchangePrel.java
@@ -89,9 +89,7 @@ public class BroadcastExchangePrel extends ExchangePrel{
}
BroadcastExchange g = new BroadcastExchange(childPOP);
- g.setOperatorId(creator.getOperatorId(this));
-
- return g;
+ return creator.addMetadata(this, g);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScan.java
index 21956dd..ce3227d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScan.java
@@ -99,7 +99,7 @@ public class ConvertCountToDirectScan extends Prule {
// 2) No GroupBY key,
// 3) only one agg function (Check if it's count(*) below).
// 4) No distinct agg call.
- if (! (oldGrpScan.getProperty().hasExactRowCount()
+ if (! (oldGrpScan.getScanStats().getGroupScanProperty().hasExactRowCount()
&& agg.getGroupCount() == 0
&& agg.getAggCallList().size() == 1
&& !agg.containsDistinctCall())) {
@@ -116,7 +116,7 @@ public class ConvertCountToDirectScan extends Prule {
if (aggCall.getArgList().isEmpty() ||
(aggCall.getArgList().size() == 1 &&
! agg.getChild().getRowType().getFieldList().get(aggCall.getArgList().get(0).intValue()).getType().isNullable())) {
- cnt = oldGrpScan.getSize().getRecordCount();
+ cnt = (long) oldGrpScan.getScanStats().getRecordCount();
} else if (aggCall.getArgList().size() == 1) {
// count(columnName) ==> Agg ( Scan )) ==> columnValueCount
int index = aggCall.getArgList().get(0);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
index 1b13dcd..c857261 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
@@ -57,8 +57,7 @@ public class FilterPrel extends DrillFilterRelBase implements Prel {
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
Filter p = new Filter(childPOP, getFilterExpression(new DrillParseContext()), 1.0f);
- p.setOperatorId(creator.getOperatorId(this));
- return p;
+ return creator.addMetadata(this, p);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
index 6ea1352..34c0b5a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
@@ -97,9 +97,7 @@ public class HashAggPrel extends AggPrelBase implements Prel{
aggExprs.toArray(new NamedExpression[aggExprs.size()]),
1.0f);
- g.setOperatorId(creator.getOperatorId(this));
-
- return g;
+ return creator.addMetadata(this, g);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java
index 9ac0834..3af1bf4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashJoinPrel.java
@@ -102,9 +102,7 @@ public class HashJoinPrel extends JoinPrel {
}
HashJoinPOP hjoin = new HashJoinPOP(leftPop, rightPop, conditions, jtype);
- hjoin.setOperatorId(creator.getOperatorId(this));
-
- return hjoin;
+ return creator.addMetadata(this, hjoin);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToMergeExchangePrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToMergeExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToMergeExchangePrel.java
index 0539a33..4518e17 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToMergeExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToMergeExchangePrel.java
@@ -90,8 +90,8 @@ public class HashToMergeExchangePrel extends ExchangePrel {
HashToMergeExchange g = new HashToMergeExchange(childPOP,
PrelUtil.getHashExpression(this.distFields, getChild().getRowType()),
PrelUtil.getOrdering(this.collation, getChild().getRowType()));
- g.setOperatorId(creator.getOperatorId(this));
- return g;
+ return creator.addMetadata(this, g);
+
}
public List<DistributionField> getDistFields() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToRandomExchangePrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToRandomExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToRandomExchangePrel.java
index b9a021e..a69cf5f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToRandomExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashToRandomExchangePrel.java
@@ -94,9 +94,7 @@ public class HashToRandomExchangePrel extends ExchangePrel {
if(PrelUtil.getSettings(getCluster()).isSingleMode()) return childPOP;
HashToRandomExchange g = new HashToRandomExchange(childPOP, PrelUtil.getHashExpression(this.fields, getChild().getRowType()));
- g.setOperatorId(creator.getOperatorId(this));
-
- return g;
+ return creator.addMetadata(this, g);
}
public List<DistributionField> getFields() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
index 376f334..7fad0d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
@@ -66,9 +66,7 @@ public class LimitPrel extends DrillLimitRelBase implements Prel {
Integer last = fetch != null ? Math.max(0, RexLiteral.intValue(fetch)) + first : null;
Limit limit = new Limit(childPOP, first, last);
- limit.setOperatorId(creator.getOperatorId(this));
-
- return limit;
+ return creator.addMetadata(this, limit);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java
index ee9480d..ee7d76c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java
@@ -104,9 +104,7 @@ public class MergeJoinPrel extends JoinPrel {
}
MergeJoinPOP mjoin = new MergeJoinPOP(leftPop, rightPop, conditions, jtype);
- mjoin.setOperatorId(creator.getOperatorId(this));
-
- return mjoin;
+ return creator.addMetadata(this, mjoin);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
index 35841ab..bf1e51a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
@@ -53,9 +53,15 @@ public class PhysicalPlanCreator {
return context;
}
- public int getOperatorId(Prel prel){
- OpId id = opIdMap.get(prel);
- return id.getAsSingleInt();
+// public int getOperatorId(Prel prel){
+// OpId id = opIdMap.get(prel);
+// return id.getAsSingleInt();
+// }
+
+ public PhysicalOperator addMetadata(Prel originalPrel, PhysicalOperator op){
+ op.setOperatorId(opIdMap.get(originalPrel).getAsSingleInt());
+ op.setCost(originalPrel.computeSelfCost(originalPrel.getCluster().getPlanner()).getRows());
+ return op;
}
public PhysicalPlan build(Prel rootPrel, boolean forceRebuild) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 1b76e2b..edad125 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.planner.physical;
import net.hydromatic.optiq.tools.FrameworkContext;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValidator;
import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
@@ -95,6 +96,10 @@ public class PlannerSettings implements FrameworkContext{
return options.getOption(BROADCAST_THRESHOLD.getOptionName()).num_val;
}
+ public long getSliceTarget(){
+ return options.getOption(ExecConstants.SLICE_TARGET).num_val;
+ }
+
@Override
public <T> T unwrap(Class<T> clazz) {
if(clazz == PlannerSettings.class){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
index 1b4eb38..a27626a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
@@ -37,5 +37,4 @@ public interface Prel extends DrillRelNode, Iterable<Prel>{
public SelectionVectorMode[] getSupportedEncodings();
public SelectionVectorMode getEncoding();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
index 88cbb50..5280469 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
@@ -58,9 +58,7 @@ public class ProjectPrel extends DrillProjectRelBase implements Prel{
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
Project p = new Project(this.getProjectExpressions(new DrillParseContext()), childPOP);
- p.setOperatorId(creator.getOperatorId(this));
-
- return p;
+ return creator.addMetadata(this, p);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
index 972e47a..9bc71c0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
@@ -24,26 +24,18 @@ import java.util.List;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
-import org.eigenbase.rel.AbstractRelNode;
-import org.apache.drill.exec.planner.common.DrillScanRelBase;
-import org.apache.drill.exec.planner.cost.DrillCostBase;
+import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.apache.drill.exec.store.StoragePlugin;
+import org.eigenbase.rel.AbstractRelNode;
import org.eigenbase.rel.RelNode;
import org.eigenbase.rel.RelWriter;
-import org.eigenbase.rel.metadata.RelMetadataQuery;
import org.eigenbase.relopt.RelOptCluster;
import org.eigenbase.relopt.RelOptCost;
import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelOptTable;
import org.eigenbase.relopt.RelTraitSet;
import org.eigenbase.reltype.RelDataType;
@@ -84,8 +76,7 @@ public class ScanPrel extends AbstractRelNode implements DrillScanPrel {
@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator)
throws IOException {
- groupScan.setOperatorId(creator.getOperatorId(this));
- return groupScan;
+ return creator.addMetadata(this, groupScan);
}
@Override
@@ -110,21 +101,20 @@ public class ScanPrel extends AbstractRelNode implements DrillScanPrel {
@Override
public double getRows() {
- return this.groupScan.getSize().getRecordCount();
+ return this.groupScan.getScanStats().getRecordCount();
}
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner) {
- Size scanSize = this.groupScan.getSize();
+ ScanStats stats = this.groupScan.getScanStats();
int columnCount = this.getRowType().getFieldCount();
if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
- OperatorCost scanCost = this.groupScan.getCost();
- return planner.getCostFactory().makeCost(scanSize.getRecordCount() * columnCount, scanCost.getCpu(), scanCost.getDisk());
+ return planner.getCostFactory().makeCost(stats.getRecordCount() * columnCount, stats.getCpuCost(), stats.getDiskCost());
}
// double rowCount = RelMetadataQuery.getRowCount(this);
- double rowCount = scanSize.getRecordCount();
+ double rowCount = stats.getRecordCount();
double cpuCost = rowCount * columnCount; // for now, assume cpu cost is proportional to row count.
// Even though scan is reading from disk, in the currently generated plans all plans will
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java
index 37823ab..c713d7c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java
@@ -52,9 +52,7 @@ public class ScreenPrel extends DrillScreenRelBase implements Prel {
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
Screen s = new Screen(childPOP, creator.getContext().getCurrentEndpoint());
- s.setOperatorId(creator.getOperatorId(this));
-
- return s;
+ return creator.addMetadata(this, s);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
index 4332188..042809f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
@@ -47,9 +47,7 @@ public class SelectionVectorRemoverPrel extends SinglePrel{
@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
SelectionVectorRemover r = new SelectionVectorRemover( ((Prel)getChild()).getPhysicalOperator(creator));
- r.setOperatorId(creator.getOperatorId(this));
- return r;
-
+ return creator.addMetadata(this, r);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
index 99d99a7..bea5500 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
@@ -89,9 +89,7 @@ public class SingleMergeExchangePrel extends ExchangePrel {
if(PrelUtil.getSettings(getCluster()).isSingleMode()) return childPOP;
SingleMergeExchange g = new SingleMergeExchange(childPOP, PrelUtil.getOrdering(this.collation, getChild().getRowType()));
- g.setOperatorId(creator.getOperatorId(this));
-
- return g;
+ return creator.addMetadata(this, g);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
index 3e1bcac..2a8c7cf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
@@ -74,9 +74,7 @@ public class SortPrel extends SortRel implements Prel {
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
Sort g = new ExternalSort(childPOP, PrelUtil.getOrdering(this.collation, getChild().getRowType()), false);
- g.setOperatorId(creator.getOperatorId(this));
-
- return g;
+ return creator.addMetadata(this, g);
}
public SortPrel copy(
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
index a3826ea..eaef94b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
@@ -85,8 +85,7 @@ public class StreamAggPrel extends AggPrelBase implements Prel{
StreamingAggregate g = new StreamingAggregate(child.getPhysicalOperator(creator), keys.toArray(new NamedExpression[keys.size()]),
aggExprs.toArray(new NamedExpression[aggExprs.size()]), 1.0f);
- g.setOperatorId(creator.getOperatorId(this));
- return g;
+ return creator.addMetadata(this, g);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
index 0067926..997c1c7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
@@ -61,9 +61,7 @@ public class TopNPrel extends SinglePrel {
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
TopN topN = new TopN(childPOP, PrelUtil.getOrdering(this.collation, getChild().getRowType()), false, this.limit);
- topN.setOperatorId(creator.getOperatorId(this));
-
- return topN;
+ return creator.addMetadata(this, topN);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
index 20722e9..33abb48 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
@@ -70,15 +70,13 @@ public class UnionAllPrel extends UnionPrel {
@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
List<PhysicalOperator> inputPops = Lists.newArrayList();
-
+
for (int i = 0; i < this.getInputs().size(); i++) {
inputPops.add( ((Prel)this.getInputs().get(i)).getPhysicalOperator(creator));
}
UnionAll unionall = new UnionAll(inputPops.toArray(new PhysicalOperator[inputPops.size()]));
- unionall.setOperatorId(creator.getOperatorId(this));
-
- return unionall;
+ return creator.addMetadata(this, unionall);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
index ad63906..3fa5659 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
@@ -70,16 +70,14 @@ public class UnionDistinctPrel extends UnionPrel {
@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
List<PhysicalOperator> inputPops = Lists.newArrayList();
-
+
for (int i = 0; i < this.getInputs().size(); i++) {
inputPops.add( ((Prel)this.getInputs().get(i)).getPhysicalOperator(creator));
}
///TODO: change this to UnionDistinct once implemented end-to-end..
UnionAll unionAll = new UnionAll(inputPops.toArray(new PhysicalOperator[inputPops.size()]));
- unionAll.setOperatorId(creator.getOperatorId(this));
-
- return unionAll;
+ return creator.addMetadata(this, unionAll);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
index f14df72..3fd6e32 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
@@ -81,8 +81,7 @@ public class UnionExchangePrel extends ExchangePrel {
if(PrelUtil.getSettings(getCluster()).isSingleMode()) return childPOP;
UnionExchange g = new UnionExchange(childPOP);
- g.setOperatorId(creator.getOperatorId(this));
- return g;
+ return creator.addMetadata(this, g);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java
index 70d0b1b..d4d135c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java
@@ -53,9 +53,7 @@ public class WriterPrel extends DrillWriterRelBase implements Prel {
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
Prel child = (Prel) this.getChild();
PhysicalOperator g = getCreateTableEntry().getWriter(child.getPhysicalOperator(creator));
- g.setOperatorId(creator.getOperatorId(this));
-
- return g;
+ return creator.addMetadata(this, g);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java
index e3cfecc..81f247d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java
@@ -20,11 +20,18 @@ package org.apache.drill.exec.planner.physical.visitor;
import org.apache.drill.exec.planner.physical.ExchangePrel;
import org.apache.drill.exec.planner.physical.JoinPrel;
import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.planner.physical.ScreenPrel;
import org.apache.drill.exec.planner.physical.WriterPrel;
public class BasePrelVisitor<RETURN, EXTRA, EXCEP extends Throwable> implements PrelVisitor<RETURN, EXTRA, EXCEP> {
+
+ @Override
+ public RETURN visitScan(ScanPrel prel, EXTRA value) throws EXCEP {
+ return visitPrel(prel, value);
+ }
+
@Override
public RETURN visitExchange(ExchangePrel prel, EXTRA value) throws EXCEP {
return visitPrel(prel, value);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
new file mode 100644
index 0000000..ae4d661
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical.visitor;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.exec.planner.physical.ExchangePrel;
+import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.ScanPrel;
+import org.apache.drill.exec.planner.physical.ScreenPrel;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCost;
+
+import com.google.common.collect.Lists;
+
+public class ExcessiveExchangeIdentifier extends BasePrelVisitor<Prel, ExcessiveExchangeIdentifier.MajorFragmentStat, RuntimeException> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExcessiveExchangeIdentifier.class);
+
+ private final long targetSliceSize;
+
+ public ExcessiveExchangeIdentifier(long targetSliceSize){
+ this.targetSliceSize = targetSliceSize;
+ }
+
+ public static Prel removeExcessiveEchanges(Prel prel, long targetSliceSize){
+ ExcessiveExchangeIdentifier exchange = new ExcessiveExchangeIdentifier(targetSliceSize);
+ return prel.accept(exchange, exchange.getNewStat());
+ }
+
+ @Override
+ public Prel visitExchange(ExchangePrel prel, MajorFragmentStat parent) throws RuntimeException {
+ MajorFragmentStat newFrag = new MajorFragmentStat();
+ Prel newChild = ((Prel) prel.getChild()).accept(this, newFrag);
+
+ if(newFrag.isSingular() && parent.isSingular()){
+ return newChild;
+ }else{
+ return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList((RelNode) newChild));
+ }
+ }
+
+ @Override
+ public Prel visitScreen(ScreenPrel prel, MajorFragmentStat s) throws RuntimeException {
+ s.setSingular();
+ RelNode child = ((Prel)prel.getChild()).accept(this, s);
+ return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList(child));
+ }
+
+ @Override
+ public Prel visitScan(ScanPrel prel, MajorFragmentStat s) throws RuntimeException {
+ s.addScan(prel);
+ return prel;
+ }
+
+
+ @Override
+ public Prel visitPrel(Prel prel, MajorFragmentStat s) throws RuntimeException {
+ List<RelNode> children = Lists.newArrayList();
+ s.add(prel);
+ for(Prel p : prel){
+ children.add(p.accept(this, s));
+ }
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+ }
+
+ public MajorFragmentStat getNewStat(){
+ return new MajorFragmentStat();
+ }
+
+ class MajorFragmentStat{
+ private double maxRows = 0d;
+ private int maxWidth = Integer.MAX_VALUE;
+
+ public void add(Prel prel){
+ RelOptCost cost = prel.computeSelfCost(prel.getCluster().getPlanner());
+ maxRows = Math.max(cost.getRows(), maxRows);
+ }
+
+ public void setSingular(){
+ maxWidth = 1;
+ }
+
+ public void addScan(ScanPrel prel){
+ maxWidth = Math.min(maxWidth, prel.getGroupScan().getMaxParallelizationWidth());
+ add(prel);
+ }
+
+ public boolean isSingular(){
+ int suggestedWidth = (int) Math.ceil((maxRows+1)/targetSliceSize);
+
+ int w = Math.min(maxWidth, suggestedWidth);
+ if(w < 1) w = 1;
+ return w == 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java
index b834464..f519220 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.planner.physical.visitor;
import org.apache.drill.exec.planner.physical.ExchangePrel;
import org.apache.drill.exec.planner.physical.JoinPrel;
import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.planner.physical.ScreenPrel;
import org.apache.drill.exec.planner.physical.WriterPrel;
@@ -30,7 +31,7 @@ public interface PrelVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitExchange(ExchangePrel prel, EXTRA value) throws EXCEP;
public RETURN visitScreen(ScreenPrel prel, EXTRA value) throws EXCEP;
public RETURN visitWriter(WriterPrel prel, EXTRA value) throws EXCEP;
-
+ public RETURN visitScan(ScanPrel prel, EXTRA value) throws EXCEP;
public RETURN visitJoin(JoinPrel prel, EXTRA value) throws EXCEP;
public RETURN visitPrel(Prel prel, EXTRA value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index aca0ae6..78dadbf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
import org.apache.drill.common.logical.PlanProperties.PlanPropertiesBuilder;
import org.apache.drill.common.logical.PlanProperties.PlanType;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.AbstractPhysicalVisitor;
@@ -41,8 +42,9 @@ import org.apache.drill.exec.planner.physical.DrillDistributionTrait;
import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
-import org.apache.drill.exec.planner.physical.visitor.FlattenPrelVisitor;
+import org.apache.drill.exec.planner.physical.visitor.ExcessiveExchangeIdentifier;
import org.apache.drill.exec.planner.physical.visitor.FinalColumnReorderer;
+import org.apache.drill.exec.planner.physical.visitor.FlattenPrelVisitor;
import org.apache.drill.exec.planner.physical.visitor.JoinPrelRenameVisitor;
import org.apache.drill.exec.planner.physical.visitor.RelUniqifier;
import org.apache.drill.exec.planner.physical.visitor.SelectionVectorPrelVisitor;
@@ -64,6 +66,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
protected final Planner planner;
protected final QueryContext context;
private Pointer<String> textPlan;
+ private final long targetSliceSize;
public DefaultSqlHandler(Planner planner, QueryContext context) {
this(planner, context, null);
@@ -74,6 +77,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
this.planner = planner;
this.context = context;
this.textPlan = textPlan;
+ targetSliceSize = context.getOptions().getOption(ExecConstants.SLICE_TARGET).num_val;
}
protected void log(String name, RelNode node) {
@@ -158,14 +162,22 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
*/
phyRelNode = FinalColumnReorderer.addFinalColumnOrdering(phyRelNode);
- /* 3.)
+ /*
+ * 3.)
+ * If two fragments are both estimated to be parallelization one, remove the exchange
+ * separating them
+ */
+ phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize);
+
+
+ /* 4.)
* Next, we add any required selection vector removers given the supported encodings of each
* operator. This will ultimately move to a new trait but we're managing here for now to avoid
* introducing new issues in planning before the next release
*/
phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
- /* 4.)
+ /* 5.)
* if the client does not support complex types (Map, Repeated)
* insert a project which which would convert
*/
@@ -174,11 +186,11 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
phyRelNode = FlattenPrelVisitor.addFlattenPrel(phyRelNode);
}
- /* 5.)
+ /* 6.)
* Finally, Make sure that the no rels are repeats.
* This could happen in the case of querying the same table twice as Optiq may canonicalize these.
*/
- phyRelNode = RelUniqifier.uniqifyGraph(phyRelNode);
+ phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize);
return phyRelNode;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/WorkEventBus.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/WorkEventBus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/WorkEventBus.java
index a05041f..bc10d4b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/WorkEventBus.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/WorkEventBus.java
@@ -90,8 +90,8 @@ public class WorkEventBus {
if (manager != null) return manager;
DistributedMap<FragmentHandle, PlanFragment> planCache = bee.getContext().getCache().getMap(Foreman.FRAGMENT_CACHE);
for(Map.Entry<FragmentHandle, PlanFragment> e : planCache.getLocalEntries()){
- logger.debug("Key: {}", e.getKey());
- logger.debug("Value: {}", e.getValue());
+// logger.debug("Key: {}", e.getKey());
+// logger.debug("Value: {}", e.getValue());
}
PlanFragment fragment = bee.getContext().getCache().getMap(Foreman.FRAGMENT_CACHE).get(handle);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 898438b..c950c5f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -25,11 +25,13 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.DistributedCache.CacheConfig;
+import org.apache.drill.exec.planner.fragment.SimpleParallelizer;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
import org.apache.drill.exec.store.sys.PStore;
import org.apache.drill.exec.store.sys.PStoreConfig;
import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.work.foreman.Foreman;
import org.eigenbase.sql.SqlLiteral;
import com.google.common.collect.Maps;
@@ -52,7 +54,17 @@ public class SystemOptionManager implements OptionManager{
PlannerSettings.BROADCAST,
PlannerSettings.BROADCAST_THRESHOLD,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
- ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR
+ ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
+ ExecConstants.SLICE_TARGET_OPTION,
+ ExecConstants.AFFINITY_FACTOR,
+ ExecConstants.MAX_WIDTH_GLOBAL,
+ ExecConstants.MAX_WIDTH_PER_NODE,
+ ExecConstants.ENABLE_QUEUE,
+ ExecConstants.LARGE_QUEUE_SIZE,
+ ExecConstants.QUEUE_THRESHOLD_SIZE,
+ ExecConstants.QUEUE_TIMEOUT,
+ ExecConstants.SMALL_QUEUE_SIZE
+
};
public final PStoreConfig<OptionValue> config;
@@ -167,7 +179,7 @@ public class SystemOptionManager implements OptionManager{
@Override
public OptionValue validate(String name, SqlLiteral value) throws SetOptionException {
OptionValidator validator = knownOptions.get(name);
- if(validator == null) throw new SetOptionException("Unknown option " + name);
+ if(validator == null) throw new SetOptionException("Unknown option: " + name);
return validator.validate(value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
index fa219e7..3de99c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
@@ -26,11 +26,11 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
+import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.dfs.FileSelection;
@@ -40,7 +40,6 @@ import org.apache.drill.exec.store.schedule.BlockMapBuilder;
import org.apache.drill.exec.store.schedule.CompleteFileWork;
import org.apache.drill.exec.store.schedule.CompleteFileWork.FileWorkImpl;
-import com.google.common.collect.Lists;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -48,6 +47,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.base.Preconditions;
import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
@JsonTypeName("fs-scan")
public class EasyGroupScan extends AbstractGroupScan{
@@ -130,16 +130,16 @@ public class EasyGroupScan extends AbstractGroupScan{
return maxWidth;
}
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1,1,1,1);
- }
@Override
- public Size getSize() {
- int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size();
- int avgColumnSize = 10;
- return new Size(1024, numColumns*avgColumnSize);
+ public ScanStats getScanStats() {
+ long data =0;
+ for(CompleteFileWork work : chunks){
+ data += work.getTotalBytes();
+ }
+
+ long estRowCount = data/1024;
+ return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, estRowCount, 1, data);
}
@JsonProperty("files")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
index 5ca781b..e12c5b3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.store.StoragePluginRegistry;
@@ -80,7 +79,7 @@ public class EasyWriter extends AbstractWriter {
}
@JsonIgnore
- public EasyFormatPlugin getFormatPlugin(){
+ public EasyFormatPlugin<?> getFormatPlugin(){
return formatPlugin;
}
@@ -90,12 +89,6 @@ public class EasyWriter extends AbstractWriter {
}
@Override
- public OperatorCost getCost() {
- // TODO:
- return new OperatorCost(1,1,1,1);
- }
-
- @Override
public int getOperatorType() {
return formatPlugin.getReaderOperatorType();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
index bcf5984..122e60d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java
@@ -24,11 +24,11 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
+import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.store.RecordReader;
@@ -59,14 +59,8 @@ public class DirectGroupScan extends AbstractGroupScan{
return 1;
}
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1,1,1,1);
- }
-
- @Override
- public Size getSize() {
- return new Size(1,1);
+ public ScanStats getScanStats(){
+ return ScanStats.TRIVIAL_TABLE;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
index 855b05f..32f793e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -20,8 +20,7 @@ package org.apache.drill.exec.store.hive;
import java.util.List;
import net.hydromatic.optiq.Schema.TableType;
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.Size;
+
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
index 504348d..2f217d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
@@ -30,15 +30,16 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.schedule.CompleteFileWork;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -254,16 +255,18 @@ public class HiveScan extends AbstractGroupScan {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(1, 2, 1, 1);
- }
+ public ScanStats getScanStats() {
+ try {
+ long data =0;
+ for(InputSplit split : inputSplits){
+ data += split.getLength();
+ }
- @Override
- public Size getSize() {
- // TODO - this is wrong, need to populate correctly
- int avgColumnSize = 10;
- int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size();
- return new Size(10, avgColumnSize*numColumns);
+ long estRowCount = data/1024;
+ return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, estRowCount, 1, data);
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
index 191af6b..c0e479a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
@@ -24,11 +24,9 @@ import java.util.List;
import org.apache.commons.codec.binary.Base64;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.hadoop.fs.Path;
@@ -122,17 +120,6 @@ public class HiveSubScan extends AbstractBase implements SubScan {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(1, 2, 1, 1);
- }
-
- @Override
- public Size getSize() {
- // TODO - this is wrong, need to populate correctly
- return new Size(10,10);
- }
-
- @Override
public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
return physicalVisitor.visitSubScan(this, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
index 01e5a60..5a9a74c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
@@ -24,11 +24,10 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
@@ -72,21 +71,13 @@ public class InfoSchemaGroupScan extends AbstractGroupScan{
return new InfoSchemaSubScan(table);
}
- @Override
- public int getMaxParallelizationWidth() {
- return 1;
- }
-
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1,1,1,1);
+ public ScanStats getScanStats(){
+ return ScanStats.TRIVIAL_TABLE;
}
@Override
- public Size getSize() {
- int avgColumnSize = 10;
- int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size();
- return new Size(1000, numColumns*avgColumnSize);
+ public int getMaxParallelizationWidth() {
+ return 1;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
index 84b6690..80515fa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
@@ -22,17 +22,18 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
+import javax.validation.constraints.Size;
+
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
@@ -50,24 +51,19 @@ public class MockGroupScanPOP extends AbstractGroupScan {
private final String url;
protected final List<MockScanEntry> readEntries;
- private final OperatorCost cost;
- private final Size size;
private LinkedList<MockScanEntry>[] mappings;
@JsonCreator
public MockGroupScanPOP(@JsonProperty("url") String url, @JsonProperty("entries") List<MockScanEntry> readEntries) {
this.readEntries = readEntries;
- OperatorCost cost = new OperatorCost(0,0,0,0);
- Size size = new Size(0,0);
- for(MockScanEntry r : readEntries){
- cost = cost.add(r.getCost());
- size = size.add(r.getSize());
- }
- this.cost = cost;
- this.size = size;
+
this.url = url;
}
+ public ScanStats getScanStats(){
+ return ScanStats.TRIVIAL_TABLE;
+ }
+
public String getUrl() {
return url;
}
@@ -95,11 +91,6 @@ public class MockGroupScanPOP extends AbstractGroupScan {
this.recordSize = size;
}
- @JsonIgnore
- public OperatorCost getCost() {
- return new OperatorCost(1, 2, 1, 1);
- }
-
public int getRecords() {
return records;
}
@@ -108,11 +99,6 @@ public class MockGroupScanPOP extends AbstractGroupScan {
return types;
}
- @JsonIgnore
- public Size getSize() {
- return new Size(records, recordSize);
- }
-
@Override
public String toString() {
return "MockScanEntry [records=" + records + ", columns=" + Arrays.toString(types) + "]";
@@ -214,16 +200,6 @@ public class MockGroupScanPOP extends AbstractGroupScan {
}
@Override
- public OperatorCost getCost() {
- return cost;
- }
-
- @Override
- public Size getSize() {
- return size;
- }
-
- @Override
@JsonIgnore
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
Preconditions.checkArgument(children.isEmpty());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorePOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorePOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorePOP.java
index 869e40c..4c12d57 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorePOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorePOP.java
@@ -21,12 +21,10 @@ import java.util.Collections;
import java.util.List;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractStore;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Store;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -61,11 +59,6 @@ public class MockStorePOP extends AbstractStore {
}
@Override
- public OperatorCost getCost() {
- return new OperatorCost(1,getSize().getRecordCount()*getSize().getRecordSize(),1,1);
- }
-
- @Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
return new MockStorePOP(child);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
index 517ad3e..705452d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
@@ -21,11 +21,9 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
@@ -74,16 +72,6 @@ public class MockSubScanPOP extends AbstractBase implements SubScan {
return Iterators.emptyIterator();
}
- @Override @JsonIgnore
- public OperatorCost getCost() {
- throw new UnsupportedOperationException();
- }
-
- @Override @JsonIgnore
- public Size getSize() {
- throw new UnsupportedOperationException();
- }
-
// will want to replace these two methods with an interface above for AbstractSubScan
@Override
public boolean isExecutable() {
[28/32] git commit: DRILL-958: Fix Parquet bug with fixed width
nullable column.
Posted by ja...@apache.org.
DRILL-958: Fix Parquet bug with fixed width nullable column.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c022b403
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c022b403
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c022b403
Branch: refs/heads/master
Commit: c022b403e8fef15797f0ad6b9290a304f6de0c5c
Parents: 9e7b14a
Author: Jason Altekruse <al...@gmail.com>
Authored: Sat Jun 14 22:30:16 2014 -0500
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:32 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/parquet/NullableColumnReader.java | 3 ++-
.../exec/store/parquet/ParquetRecordReaderTest.java | 11 +++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c022b403/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
index eeb0344..585fd66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
@@ -63,7 +63,7 @@ abstract class NullableColumnReader<V extends ValueVector> extends ColumnReader<
// are located and copied together, rather than copying individual values
long runStart = pageReadStatus.readPosInBytes;
- int runLength = 0;
+ int runLength;
int currentDefinitionLevel = 0;
int currentValueIndexInVector = (int) recordsReadInThisIteration;
boolean lastValueWasNull = true;
@@ -73,6 +73,7 @@ abstract class NullableColumnReader<V extends ValueVector> extends ColumnReader<
definitionLevelsRead = 0;
lastValueWasNull = true;
nullsFound = 0;
+ runLength = 0;
if (currentValueIndexInVector == recordsToReadInThisPass
|| currentValueIndexInVector >= valueVec.getValueCapacity()) {
break;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c022b403/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index ad63dc9..fbe037b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -392,6 +392,17 @@ public class ParquetRecordReaderTest extends BaseTestQuery{
"unused, no file is generated", 1, props, true);
}
+ @Ignore
+ @Test
+ public void drill_958bugTest() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
+ TestFileGenerator.populatePigTPCHCustomerFields(props);
+ String readEntries = "\"/tmp/store_sales\"";
+ testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ "unused, no file is generated", 1, props, true);
+ }
+
@Test
public void testMultipleRowGroupsEvent() throws Exception {
HashMap<String, FieldInfo> fields = new HashMap<>();
[13/32] git commit: DRILL-980: Add profile summary to web ui
Posted by ja...@apache.org.
DRILL-980: Add profile summary to web ui
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/49a9ff27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/49a9ff27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/49a9ff27
Branch: refs/heads/master
Commit: 49a9ff27f283cbc1c8749989ff408440a0275e7d
Parents: 6801a5d
Author: Steven Phillips <sp...@maprtech.com>
Authored: Fri Jun 13 00:17:47 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:02:36 2014 -0700
----------------------------------------------------------------------
.../exec/server/rest/ProfileResources.java | 4 +-
.../drill/exec/server/rest/ProfileWrapper.java | 148 +++++++++++++++++++
.../drill/exec/work/foreman/FragmentData.java | 2 +-
.../work/fragment/AbstractStatusReporter.java | 1 +
.../src/main/resources/rest/profile/profile.ftl | 10 +-
5 files changed, 160 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
index 386db46..7d29ea3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileResources.java
@@ -57,7 +57,9 @@ public class ProfileResources {
QueryProfile profile = profiles.get(queryId);
if(profile == null) profile = QueryProfile.getDefaultInstance();
- return new Viewable("/rest/profile/profile.ftl", profile);
+ ProfileWrapper wrapper = new ProfileWrapper(profile);
+
+ return new Viewable("/rest/profile/profile.ftl", wrapper);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
new file mode 100644
index 0000000..a1d4df9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ProfileWrapper.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile;
+import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile;
+import org.apache.drill.exec.proto.UserBitShared.OperatorProfile;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+
+import java.text.NumberFormat;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+public class ProfileWrapper {
+
+ NumberFormat format = NumberFormat.getInstance(Locale.US);
+
+ public QueryProfile profile;
+
+ public ProfileWrapper(QueryProfile profile) {
+ this.profile = profile;
+ }
+
+ public QueryProfile getProfile() {
+ return profile;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("MAJOR FRAGMENTS\nid\tmin\tavg\tmax\t(time in ms)\n\n" + listMajorFragments());
+ builder.append("\n");
+ for (MajorFragmentProfile majorProfile : profile.getFragmentProfileList()) {
+ builder.append(String.format("Major Fragment: %d\n%s\n", majorProfile.getMajorFragmentId(), new MajorFragmentWrapper(majorProfile).toString()));
+ }
+ return builder.toString();
+ }
+
+ public String listMajorFragments() {
+ StringBuilder builder = new StringBuilder();
+ for (MajorFragmentProfile m : profile.getFragmentProfileList()) {
+ List<Long> totalTimes = Lists.newArrayList();
+ for (MinorFragmentProfile minorFragmentProfile : m.getMinorFragmentProfileList()) {
+ totalTimes.add(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime());
+ }
+ long min = Collections.min(totalTimes);
+ long max = Collections.max(totalTimes);
+ long sum = 0;
+ for (Long l : totalTimes) {
+ sum += l;
+ }
+ long avg = sum / totalTimes.size();
+ builder.append(String.format("%d\t%s\t%s\t%s\n", m.getMajorFragmentId(), format.format(min), format.format(avg), format.format(max)));
+ }
+ return builder.toString();
+ }
+
+ public class MajorFragmentWrapper {
+ MajorFragmentProfile majorFragmentProfile;
+
+ public MajorFragmentWrapper(MajorFragmentProfile majorFragmentProfile) {
+ this.majorFragmentProfile = majorFragmentProfile;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Minor Fragments\nid\ttotal time (ms)\n%s\nOperators\nid\ttype\tmin\tavg\tmax\t(time in ns)\n%s\n", new MinorFragmentsInMajor().toString(), new OperatorsInMajor().toString());
+ }
+
+ public class MinorFragmentsInMajor {
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ for (MinorFragmentProfile minorFragmentProfile: majorFragmentProfile.getMinorFragmentProfileList()) {
+ builder.append(String.format("%d\t%s\n", minorFragmentProfile.getMinorFragmentId(), format.format(minorFragmentProfile.getEndTime() - minorFragmentProfile.getStartTime())));
+ }
+ return builder.toString();
+ }
+ }
+
+ public class OperatorsInMajor {
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ int numOperators = majorFragmentProfile.getMinorFragmentProfile(0).getOperatorProfileCount();
+ int numFragments = majorFragmentProfile.getMinorFragmentProfileCount();
+ long[][] values = new long[numOperators + 1][numFragments];
+ CoreOperatorType[] operatorTypes = new CoreOperatorType[numOperators + 1];
+
+ for (int i = 0; i < numFragments; i++) {
+ MinorFragmentProfile minorProfile = majorFragmentProfile.getMinorFragmentProfile(i);
+ for (int j = 0; j < numOperators; j++) {
+ OperatorProfile operatorProfile = minorProfile.getOperatorProfile(j);
+ int operatorId = operatorProfile.getOperatorId();
+ values[operatorId][i] = operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos();
+ if (i == 0) {
+ operatorTypes[operatorId] = CoreOperatorType.valueOf(operatorProfile.getOperatorType());
+ }
+ }
+ }
+
+ for (int j = 0; j < numOperators + 1; j++) {
+ if (operatorTypes[j] == null) {
+ continue;
+ }
+ long min = Long.MAX_VALUE;
+ long max = Long.MIN_VALUE;
+ long sum = 0;
+
+ for (int i = 0; i < numFragments; i++) {
+ min = Math.min(min, values[j][i]);
+ max = Math.max(max, values[j][i]);
+ sum += values[j][i];
+ }
+
+ long avg = sum / numFragments;
+
+ builder.append(String.format("%d\t%s\t%s\t%s\t%s\n", j, operatorTypes[j].toString(), format.format(min), format.format(avg), format.format(max)));
+ }
+ return builder.toString();
+ }
+ }
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
index d9dd33e..b9b3de0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentData.java
@@ -31,7 +31,7 @@ public class FragmentData {
public FragmentData(FragmentHandle handle, DrillbitEndpoint endpoint, boolean isLocal) {
super();
- MinorFragmentProfile f = MinorFragmentProfile.newBuilder().setState(FragmentState.SENDING).build();
+ MinorFragmentProfile f = MinorFragmentProfile.newBuilder().setState(FragmentState.SENDING).setMinorFragmentId(handle.getMinorFragmentId()).build();
this.status = FragmentStatus.newBuilder().setHandle(handle).setProfile(f).build();
this.endpoint = endpoint;
this.isLocal = isLocal;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
index 30e4b6a..8d9da19 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/AbstractStatusReporter.java
@@ -48,6 +48,7 @@ public abstract class AbstractStatusReporter implements StatusReporter{
}
status.setHandle(context.getHandle());
b.setMemoryUsed(context.getAllocator().getAllocatedMemory());
+ b.setMinorFragmentId(context.getHandle().getMinorFragmentId());
status.setProfile(b);
return status;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/49a9ff27/exec/java-exec/src/main/resources/rest/profile/profile.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
index ffd22db..c3149ab 100644
--- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
@@ -20,7 +20,7 @@
<h2>Query</h2>
<form role="form" action="/query" method="POST">
<div class="form-group">
- <textarea class="form-control" id="query" name="query">${model.query}</textarea>
+ <textarea class="form-control" id="query" name="query">${model.getProfile().query}</textarea>
</div>
<div class="form-group">
<div class="radio-inline">
@@ -47,11 +47,15 @@
<div class="page-header">
<h2>Physical Plan</h2>
</div>
- <p><pre>${model.plan}</pre></p>
+ <p><pre>${model.profile.plan}</pre></p>
<div class="page-header">
- <h2>Complete Profile</h2>
+ <h2>Profile Summary</h2>
</div>
<p><pre>${model.toString()}</pre></p>
+ <div class="page-header">
+ <h2>Complete Profile</h2>
+ </div>
+ <p><pre>${model.profile.toString()}</pre></p>
</#macro>
<@page_html/>
[14/32] git commit: DRILL-954: Exclude servlet-api from classpath,
and exclude duplicate servlet-api dependencies
Posted by ja...@apache.org.
DRILL-954: Exclude servlet-api from classpath, and exclude duplicate servlet-api dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6801a5dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6801a5dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6801a5dc
Branch: refs/heads/master
Commit: 6801a5dc8b48c24b8fabab63945af70787a4a3be
Parents: 989d336
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Jun 12 13:33:11 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 08:02:36 2014 -0700
----------------------------------------------------------------------
contrib/storage-hbase/pom.xml | 8 ++++++
contrib/storage-hive/pom.xml | 4 +++
distribution/src/resources/hadoop-excludes.txt | 1 +
pom.xml | 32 +++++++++++++++++++++
4 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6801a5dc/contrib/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml
index 09b4f0f..0c39b7d 100644
--- a/contrib/storage-hbase/pom.xml
+++ b/contrib/storage-hbase/pom.xml
@@ -100,6 +100,10 @@
<version>0.94.11</version>
<exclusions>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
@@ -155,6 +159,10 @@
<scope>test</scope>
<exclusions>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6801a5dc/contrib/storage-hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/pom.xml b/contrib/storage-hive/pom.xml
index 314e016..6e416e4 100644
--- a/contrib/storage-hive/pom.xml
+++ b/contrib/storage-hive/pom.xml
@@ -80,6 +80,10 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6801a5dc/distribution/src/resources/hadoop-excludes.txt
----------------------------------------------------------------------
diff --git a/distribution/src/resources/hadoop-excludes.txt b/distribution/src/resources/hadoop-excludes.txt
index d72efeb..36b5703 100644
--- a/distribution/src/resources/hadoop-excludes.txt
+++ b/distribution/src/resources/hadoop-excludes.txt
@@ -11,3 +11,4 @@ jersey
eclipse
common
guava
+servlet
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6801a5dc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2844a10..e1617df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -582,6 +582,14 @@
<version>1.2.1</version>
<exclusions>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>jets3t</artifactId>
<groupId>net.java.dev.jets3t</groupId>
</exclusion>
@@ -694,6 +702,14 @@
<version>1.0.3-mapr-3.0.2</version>
<exclusions>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>asm</artifactId>
<groupId>asm</groupId>
</exclusion>
@@ -804,6 +820,14 @@
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
@@ -831,6 +855,14 @@
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
[03/32] git commit: DRILL-968: Use checkstyle plugin to prevent
inadvertent use of shaded Guava classes
Posted by ja...@apache.org.
DRILL-968: Use checkstyle plugin to prevent inadvertent use of shaded Guava classes
+ Disallow non-static '*' imports in handwritten code.
+ Updated the current code to be in compliance.
+ Run 'rat' plugin in 'validate' phase.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5da52cb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5da52cb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5da52cb5
Branch: refs/heads/master
Commit: 5da52cb510b58f9f9022608cbdb4f7d9e2d5d368
Parents: 4198a17
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Wed Jun 11 20:09:10 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Thu Jun 12 13:02:26 2014 -0700
----------------------------------------------------------------------
.../apache/drill/common/config/DrillConfig.java | 1 -
.../drill/common/config/NestedConfig.java | 10 ++-
.../common/logical/data/LogicalOperator.java | 11 ++--
.../data/visitors/AbstractLogicalVisitor.java | 18 +++++-
.../drill/common/util/DataInputInputStream.java | 7 ++-
.../drill/common/util/DecimalUtility.java | 5 +-
.../drill/hbase/TestHBaseTableProvider.java | 9 ---
distribution/pom.xml | 1 +
.../drill/exec/cache/infinispan/ICache.java | 2 +-
.../drill/exec/compile/ClassTransformer.java | 6 +-
.../drill/exec/compile/sig/SignatureHolder.java | 4 +-
.../apache/drill/exec/expr/ClassGenerator.java | 5 +-
.../drill/exec/expr/fn/FunctionConverter.java | 2 +-
.../exec/expr/fn/impl/DateTypeFunctions.java | 15 ++---
.../drill/exec/expr/fn/impl/HashFunctions.java | 31 +++++++++-
.../apache/drill/exec/expr/fn/impl/IsFalse.java | 7 ++-
.../drill/exec/expr/fn/impl/IsNotFalse.java | 7 ++-
.../drill/exec/expr/fn/impl/IsNotTrue.java | 7 ++-
.../apache/drill/exec/expr/fn/impl/IsTrue.java | 7 ++-
.../apache/drill/exec/ops/FragmentContext.java | 3 +-
.../physical/base/AbstractPhysicalVisitor.java | 22 ++++++-
.../exec/physical/base/PhysicalVisitor.java | 22 ++++++-
.../exec/physical/config/BroadcastExchange.java | 12 ++--
.../drill/exec/physical/config/UnionAll.java | 12 ++--
.../impl/TopN/PriorityQueueTemplate.java | 16 +++--
.../exec/physical/impl/TopN/TopNBatch.java | 35 +++++++----
.../exec/physical/impl/WireRecordBatch.java | 9 ++-
.../exec/physical/impl/WriterRecordBatch.java | 10 ++-
.../physical/impl/filter/FilterRecordBatch.java | 9 +--
.../exec/physical/impl/join/MergeJoinBatch.java | 13 ++--
.../impl/join/MergeJoinBatchBuilder.java | 12 ++--
.../physical/impl/limit/LimitRecordBatch.java | 13 ++--
.../impl/mergereceiver/MergingRecordBatch.java | 22 ++++---
.../PartitionSenderRootExec.java | 13 ++--
.../partitionsender/PartitionerTemplate.java | 27 +++++----
.../physical/impl/sort/RecordBatchData.java | 6 +-
.../impl/svremover/RemovingRecordBatch.java | 16 +++--
.../impl/union/UnionAllRecordBatch.java | 15 +++--
.../exec/physical/impl/xsort/BatchGroup.java | 17 ++++--
.../physical/impl/xsort/ExternalSortBatch.java | 34 +++++++----
.../impl/xsort/PriorityQueueCopierTemplate.java | 18 ++----
.../exec/planner/fragment/Materializer.java | 6 +-
.../exec/planner/fragment/StatsCollector.java | 6 +-
.../planner/logical/DrillAggregateRule.java | 9 ++-
.../exec/planner/logical/DrillJoinRule.java | 13 ++--
.../exec/planner/logical/DrillRuleSets.java | 19 +++++-
.../exec/planner/logical/DrillSortRule.java | 7 ++-
.../exec/planner/logical/DrillUnionRule.java | 15 +++--
.../exec/planner/logical/DrillValuesRule.java | 5 +-
.../exec/planner/physical/AggPrelBase.java | 2 +-
.../sql/handlers/DescribeTableHandler.java | 16 +++--
.../sql/handlers/ShowSchemasHandler.java | 11 +++-
.../planner/sql/handlers/ShowTablesHandler.java | 18 ++++--
.../exec/planner/sql/parser/SqlCreateTable.java | 18 ++++--
.../exec/planner/sql/parser/SqlCreateView.java | 17 ++++--
.../planner/sql/parser/SqlDescribeTable.java | 15 +++--
.../exec/planner/sql/parser/SqlDropView.java | 13 +++-
.../exec/planner/sql/parser/SqlShowFiles.java | 14 +++--
.../exec/planner/sql/parser/SqlShowSchemas.java | 14 +++--
.../exec/planner/sql/parser/SqlShowTables.java | 15 +++--
.../exec/planner/sql/parser/SqlUseSchema.java | 13 +++-
.../drill/exec/rpc/control/ControlClient.java | 8 ++-
.../drill/exec/rpc/control/ControlServer.java | 6 +-
.../apache/drill/exec/rpc/data/DataClient.java | 8 ++-
.../apache/drill/exec/rpc/data/DataServer.java | 6 +-
.../apache/drill/exec/rpc/user/UserClient.java | 7 ++-
.../apache/drill/exec/rpc/user/UserServer.java | 9 ++-
.../exec/store/LocalSyncableFileSystem.java | 25 ++++++--
.../apache/drill/exec/store/VectorHolder.java | 6 +-
.../drill/exec/store/dfs/FileSystemPlugin.java | 5 +-
.../drill/exec/store/hive/HiveInputReader.java | 30 ++++-----
.../drill/exec/store/hive/HiveSubScan.java | 28 +++++----
.../apache/drill/exec/store/hive/HiveTable.java | 16 +++--
.../exec/store/hive/HiveTextRecordReader.java | 14 +++--
.../exec/store/ischema/InfoSchemaTable.java | 7 ++-
.../NullableFixedByteAlignedReaders.java | 5 +-
.../exec/store/parquet/ParquetFormatPlugin.java | 11 +++-
.../exec/store/parquet/ParquetRecordReader.java | 29 ++++++---
.../drill/exec/store/parquet/ParquetWriter.java | 10 ++-
.../exec/store/parquet/VarLenBinaryReader.java | 8 ++-
.../store/parquet/VarLengthColumnReaders.java | 13 ++--
.../store/sys/local/NoWriteLocalPStore.java | 2 +-
.../complex/impl/RepeatedMapReaderImpl.java | 64 ++------------------
.../complex/impl/SingleMapReaderImpl.java | 58 +-----------------
.../drill/exec/cache/TestWriteToDisk.java | 10 ++-
.../drill/exec/fn/impl/GeneratorFunctions.java | 7 ++-
.../drill/exec/fn/impl/TestDateFunctions.java | 16 ++---
.../drill/exec/fn/impl/TestMathFunctions.java | 12 ++--
.../physical/impl/TestExtractFunctions.java | 8 +--
.../physical/impl/TestSimpleFragmentRun.java | 11 +---
.../exec/physical/impl/TestSimpleFunctions.java | 21 ++++---
.../drill/exec/physical/impl/agg/TestAgg.java | 13 ++--
.../physical/impl/common/TestHashTable.java | 16 +----
.../physical/impl/filter/TestSimpleFilter.java | 15 ++---
.../impl/trace/TestTraceMultiRecordBatch.java | 11 ++--
.../impl/trace/TestTraceOutputDump.java | 12 ++--
.../drill/exec/pop/TestInjectionValue.java | 2 +-
.../drill/exec/record/vector/TestDateTypes.java | 33 +---------
.../drill/exec/util/MiniZooKeeperCluster.java | 21 ++++---
.../apache/drill/exec/work/batch/FileTest.java | 13 ++--
.../apache/drill/jdbc/test/TestJdbcQuery.java | 27 +--------
pom.xml | 28 ++++++++-
src/main/resources/checkstyle-config.xml | 31 ++++++++++
src/main/resources/checkstyle-suppressions.xml | 19 ++++++
104 files changed, 860 insertions(+), 598 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index 1c83040..2455dd9 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -54,7 +54,6 @@ public final class DrillConfig extends NestedConfig{
@SuppressWarnings("unchecked")
private volatile List<Queue<Object>> sinkQueues = new CopyOnWriteArrayList<Queue<Object>>(new Queue[1]);
- @SuppressWarnings("restriction")
@VisibleForTesting
public DrillConfig(Config config, boolean enableServer) {
super(config);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/common/src/main/java/org/apache/drill/common/config/NestedConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/NestedConfig.java b/common/src/main/java/org/apache/drill/common/config/NestedConfig.java
index f063add..3fd885f 100644
--- a/common/src/main/java/org/apache/drill/common/config/NestedConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/NestedConfig.java
@@ -17,12 +17,18 @@
*/
package org.apache.drill.common.config;
-import com.typesafe.config.*;
-
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigList;
+import com.typesafe.config.ConfigMergeable;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigOrigin;
+import com.typesafe.config.ConfigResolveOptions;
+import com.typesafe.config.ConfigValue;
+
abstract class NestedConfig implements Config {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NestedConfig.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
index 531e6a6..3343d4e 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
@@ -20,14 +20,17 @@ package org.apache.drill.common.logical.data;
import java.util.Collection;
import java.util.List;
-import com.fasterxml.jackson.annotation.*;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.drill.common.graph.GraphValue;
import org.apache.drill.common.logical.ValidationError;
-
import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+import com.fasterxml.jackson.annotation.JsonIdentityInfo;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.ObjectIdGenerators;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
@JsonPropertyOrder({"@id", "memo", "input"}) // op will always be first since it is wrapped.
@JsonIdentityInfo(generator = ObjectIdGenerators.IntSequenceGenerator.class, property = "@id")
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "op")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
index 0099bb9..8758444 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
@@ -17,7 +17,23 @@
*/
package org.apache.drill.common.logical.data.visitors;
-import org.apache.drill.common.logical.data.*;
+import org.apache.drill.common.logical.data.Constant;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Flatten;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.RunningAggregate;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Sequence;
+import org.apache.drill.common.logical.data.Store;
+import org.apache.drill.common.logical.data.Transform;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.common.logical.data.WindowFrame;
+import org.apache.drill.common.logical.data.Writer;
public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implements LogicalVisitor<T, X, E> {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/common/src/main/java/org/apache/drill/common/util/DataInputInputStream.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DataInputInputStream.java b/common/src/main/java/org/apache/drill/common/util/DataInputInputStream.java
index f61b301..60d37e3 100644
--- a/common/src/main/java/org/apache/drill/common/util/DataInputInputStream.java
+++ b/common/src/main/java/org/apache/drill/common/util/DataInputInputStream.java
@@ -18,9 +18,12 @@
package org.apache.drill.common.util;
-import org.apache.commons.lang3.exception.ExceptionUtils;
+import java.io.DataInput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
-import java.io.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
public class DataInputInputStream extends InputStream {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
index ae2af54..4cc80ea 100644
--- a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
@@ -17,11 +17,8 @@
*/
package org.apache.drill.common.util;
-import java.math.*;
-
-
-import io.netty.buffer.Unpooled;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
import java.math.BigDecimal;
import java.math.BigInteger;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
index c94e61f..b1ff44c 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
@@ -20,24 +20,15 @@ package org.apache.drill.hbase;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
-import java.util.Map;
import java.util.Map.Entry;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
import org.apache.drill.exec.store.hbase.config.HBasePStoreProvider;
import org.apache.drill.exec.store.sys.PStore;
import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreRegistry;
-import org.apache.hadoop.hbase.HConstants;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.hive12.common.collect.Maps;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigValueFactory;
-
public class TestHBaseTableProvider extends BaseHBaseTest {
private static HBasePStoreProvider provider;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 6f8b3a9..0a6b887 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -106,6 +106,7 @@
<exclude>**/conffiles</exclude>
<exclude>**/git.properties</exclude>
<exclude>**/control</exclude>
+ <exclude>**/*.checkstyle</exclude>
<exclude>**/*.json</exclude>
<exclude>**/hadoop-excludes.txt</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
index fb346ef..651fc04 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/infinispan/ICache.java
@@ -54,7 +54,7 @@ import org.jgroups.protocols.COUNTER;
import org.jgroups.protocols.FRAG2;
import org.jgroups.stack.ProtocolStack;
-import com.google.hive12.common.collect.Maps;
+import com.google.common.collect.Maps;
public class ICache implements DistributedCache{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ICache.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
index 3dfc4ea..25c71b1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
@@ -30,10 +30,10 @@ import org.codehaus.commons.compiler.CompileException;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.tree.ClassNode;
-import com.google.common.collect.Lists;
-import com.beust.jcommander.internal.Maps;
-import com.beust.jcommander.internal.Sets;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
public class ClassTransformer {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassTransformer.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
index 158570a..536370b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
@@ -24,10 +24,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
-import com.beust.jcommander.internal.Maps;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
public class SignatureHolder implements Iterable<CodeGeneratorMethod>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SignatureHolder.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
index 51ef003..ab3c307 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
@@ -37,10 +37,9 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference;
import org.apache.drill.exec.record.TypedFieldId;
-import com.google.common.collect.Lists;
-import com.beust.jcommander.internal.Maps;
import com.google.common.base.Preconditions;
-import com.sun.codemodel.JArray;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.sun.codemodel.JBlock;
import com.sun.codemodel.JClass;
import com.sun.codemodel.JClassAlreadyExistsException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 7f5a335..44dd49d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -43,8 +43,8 @@ import org.codehaus.janino.Parser;
import org.codehaus.janino.Scanner;
import org.mortbay.util.IO;
-import com.beust.jcommander.internal.Maps;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.io.InputSupplier;
import com.google.common.io.Resources;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
index bbda087..821db11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
@@ -19,9 +19,7 @@
package org.apache.drill.exec.expr.fn.impl;
import io.netty.buffer.ByteBuf;
-import org.apache.drill.common.expression.*;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -29,16 +27,15 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.TimeHolder;
-import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.expr.holders.TimeStampHolder;
-import org.apache.drill.exec.expr.holders.TimeStampTZHolder;
import org.apache.drill.exec.expr.holders.DateHolder;
-import org.apache.drill.exec.expr.holders.IntervalHolder;
import org.apache.drill.exec.expr.holders.IntervalDayHolder;
+import org.apache.drill.exec.expr.holders.IntervalHolder;
import org.apache.drill.exec.expr.holders.IntervalYearHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.TimeStampTZHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.exec.record.RecordBatch;
-import org.joda.time.DateTimeZone;
public class DateTypeFunctions {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 1ac4f21..7475420 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -22,7 +22,36 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.Decimal18Holder;
+import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal9Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableDateHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal18Holder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal9Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampTZHolder;
+import org.apache.drill.exec.expr.holders.NullableVar16CharHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.TimeStampTZHolder;
+import org.apache.drill.exec.expr.holders.Var16CharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.exec.record.RecordBatch;
public class HashFunctions {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsFalse.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsFalse.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsFalse.java
index 72db649..52af8cd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsFalse.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsFalse.java
@@ -19,8 +19,11 @@
package org.apache.drill.exec.expr.fn.impl;
import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.*;
-import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.record.RecordBatch;
public class IsFalse {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotFalse.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotFalse.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotFalse.java
index 7fe5f2b..65476ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotFalse.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotFalse.java
@@ -19,8 +19,11 @@
package org.apache.drill.exec.expr.fn.impl;
import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.*;
-import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.record.RecordBatch;
public class IsNotFalse {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotTrue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotTrue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotTrue.java
index 5e0bf48..3f389af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotTrue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotTrue.java
@@ -19,8 +19,11 @@
package org.apache.drill.exec.expr.fn.impl;
import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.*;
-import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.record.RecordBatch;
public class IsNotTrue {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsTrue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsTrue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsTrue.java
index d85a807..7edf539 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsTrue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsTrue.java
@@ -19,8 +19,11 @@
package org.apache.drill.exec.expr.fn.impl;
import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.*;
-import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.record.RecordBatch;
public class IsTrue {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index f72d672..2b4743d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -26,7 +26,6 @@ import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.tools.Frameworks;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.compile.ClassTransformer;
import org.apache.drill.exec.compile.QueryClassLoader;
@@ -49,7 +48,7 @@ import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.work.batch.IncomingBuffers;
import com.google.common.collect.Lists;
-import com.beust.jcommander.internal.Maps;
+import com.google.common.collect.Maps;
/**
* Contextual objects required for execution of a particular fragment.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index c7c82aa..14cfe81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -17,7 +17,27 @@
*/
package org.apache.drill.exec.physical.base;
-import org.apache.drill.exec.physical.config.*;
+import org.apache.drill.exec.physical.config.BroadcastSender;
+import org.apache.drill.exec.physical.config.Filter;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.physical.config.HashJoinPOP;
+import org.apache.drill.exec.physical.config.HashPartitionSender;
+import org.apache.drill.exec.physical.config.HashToRandomExchange;
+import org.apache.drill.exec.physical.config.IteratorValidator;
+import org.apache.drill.exec.physical.config.Limit;
+import org.apache.drill.exec.physical.config.MergeJoinPOP;
+import org.apache.drill.exec.physical.config.MergingReceiverPOP;
+import org.apache.drill.exec.physical.config.OrderedPartitionSender;
+import org.apache.drill.exec.physical.config.Project;
+import org.apache.drill.exec.physical.config.RandomReceiver;
+import org.apache.drill.exec.physical.config.RangeSender;
+import org.apache.drill.exec.physical.config.Screen;
+import org.apache.drill.exec.physical.config.SingleSender;
+import org.apache.drill.exec.physical.config.Sort;
+import org.apache.drill.exec.physical.config.StreamingAggregate;
+import org.apache.drill.exec.physical.config.Trace;
+import org.apache.drill.exec.physical.config.UnionAll;
+import org.apache.drill.exec.physical.config.UnionExchange;
public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> implements PhysicalVisitor<T, X, E> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractPhysicalVisitor.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
index 1798289..3eaf026 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
@@ -17,7 +17,27 @@
*/
package org.apache.drill.exec.physical.base;
-import org.apache.drill.exec.physical.config.*;
+import org.apache.drill.exec.physical.config.BroadcastSender;
+import org.apache.drill.exec.physical.config.Filter;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.physical.config.HashJoinPOP;
+import org.apache.drill.exec.physical.config.HashPartitionSender;
+import org.apache.drill.exec.physical.config.HashToRandomExchange;
+import org.apache.drill.exec.physical.config.IteratorValidator;
+import org.apache.drill.exec.physical.config.Limit;
+import org.apache.drill.exec.physical.config.MergeJoinPOP;
+import org.apache.drill.exec.physical.config.MergingReceiverPOP;
+import org.apache.drill.exec.physical.config.OrderedPartitionSender;
+import org.apache.drill.exec.physical.config.Project;
+import org.apache.drill.exec.physical.config.RandomReceiver;
+import org.apache.drill.exec.physical.config.RangeSender;
+import org.apache.drill.exec.physical.config.Screen;
+import org.apache.drill.exec.physical.config.SingleSender;
+import org.apache.drill.exec.physical.config.Sort;
+import org.apache.drill.exec.physical.config.StreamingAggregate;
+import org.apache.drill.exec.physical.config.Trace;
+import org.apache.drill.exec.physical.config.UnionAll;
+import org.apache.drill.exec.physical.config.UnionExchange;
/**
* Visitor class designed to traversal of a operator tree. Basis for a number of operator manipulations including fragmentation and materialization.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
index 95d8803..5e67bcc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/BroadcastExchange.java
@@ -17,19 +17,19 @@
******************************************************************************/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
+import java.util.List;
+
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.physical.base.AbstractExchange;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Receiver;
import org.apache.drill.exec.physical.base.Sender;
import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import java.util.List;
-
-import static org.apache.drill.exec.proto.CoordinationProtos.*;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("broadcast-exchange")
public class BroadcastExchange extends AbstractExchange {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
index f0ca6ad..06ff2ba 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
@@ -17,15 +17,17 @@
*/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
+import java.util.List;
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.physical.base.AbstractMultiple;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("union-all")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
index fc8c430..e0a4c92 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
@@ -17,19 +17,23 @@
*/
package org.apache.drill.exec.physical.impl.TopN;
-import com.google.common.base.Stopwatch;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
+import java.util.concurrent.TimeUnit;
+
+import javax.inject.Named;
+
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.ExpandableHyperContainer;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
-import javax.inject.Named;
-import java.util.concurrent.TimeUnit;
+import com.google.common.base.Stopwatch;
public abstract class PriorityQueueTemplate implements PriorityQueue {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PriorityQueueTemplate.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 712311f..5196279 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -17,12 +17,15 @@
*/
package org.apache.drill.exec.physical.impl.TopN;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.sun.codemodel.JConditional;
-import com.sun.codemodel.JExpr;
-import org.apache.drill.common.expression.*;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.sig.MappingSet;
@@ -32,10 +35,8 @@ import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.TopN;
@@ -43,17 +44,25 @@ import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.physical.impl.sort.SortRecordBatchBuilder;
import org.apache.drill.exec.physical.impl.svremover.Copier;
import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.ExpandableHyperContainer;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
import org.eigenbase.rel.RelFieldCollation.Direction;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
public class TopNBatch extends AbstractRecordBatch<TopN> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopNBatch.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index 1163af0..fb681ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -25,7 +25,14 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.record.RawFragmentBatchProvider;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
index 3c4bc41..2dae853 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.physical.impl;
+import java.io.IOException;
+
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -26,16 +28,18 @@ import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.Writer;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordValueAccessor;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.store.EventBasedRecordWriter;
import org.apache.drill.exec.store.RecordWriter;
import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.VarCharVector;
-import java.io.IOException;
-
/* Write the RecordBatch to the given RecordWriter. */
public class WriterRecordBatch extends AbstractRecordBatch<Writer> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WriterRecordBatch.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
index fb0e03b..d52d2e3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
@@ -20,7 +20,6 @@ package org.apache.drill.exec.physical.impl.filter;
import java.io.IOException;
import java.util.List;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
@@ -33,16 +32,18 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Filter;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractSingleRecordBatch;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
import com.google.common.collect.Lists;
-import org.apache.drill.exec.vector.allocator.VectorAllocator;
-
public class FilterRecordBatch extends AbstractSingleRecordBatch<Filter>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FilterRecordBatch.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 587dd6a..84f8354 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -22,7 +22,6 @@ import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
import java.io.IOException;
import java.util.List;
-import com.google.common.base.Preconditions;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
@@ -36,21 +35,27 @@ import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.impl.join.JoinWorker.JoinOutcome;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
import org.eigenbase.rel.JoinRelType;
+import com.google.common.base.Preconditions;
import com.sun.codemodel.JClass;
import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatchBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatchBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatchBuilder.java
index a75437c..904d38c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatchBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatchBuilder.java
@@ -17,17 +17,21 @@
*/
package org.apache.drill.exec.physical.impl.join;
-import com.google.common.collect.ArrayListMultimap;
+import java.util.List;
+
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.BufferAllocator.PreAllocator;
-import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.ValueVector;
-import java.util.List;
+import com.google.common.collect.ArrayListMultimap;
public class MergeJoinBatchBuilder {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
index 3e408bd..078c4c4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
@@ -17,17 +17,20 @@
*/
package org.apache.drill.exec.physical.impl.limit;
-import com.google.common.collect.Lists;
-import com.google.common.base.Objects;
+import java.util.List;
+
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Limit;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractSingleRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.vector.ValueVector;
-import java.util.List;
+import com.google.common.collect.Lists;
public class LimitRecordBatch extends AbstractSingleRecordBatch<Limit> {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 07a949c..25ee667 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -19,19 +19,16 @@ package org.apache.drill.exec.physical.impl.mergereceiver;
*/
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.PriorityQueue;
-import com.sun.codemodel.*;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
@@ -41,15 +38,25 @@ import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
-import org.apache.drill.exec.physical.impl.xsort.PriorityQueueCopier;
import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.ExpandableHyperContainer;
+import org.apache.drill.exec.record.RawFragmentBatch;
+import org.apache.drill.exec.record.RawFragmentBatchProvider;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.SchemaBuilder;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.CopyUtil;
@@ -59,8 +66,9 @@ import org.eigenbase.rel.RelFieldCollation.Direction;
import parquet.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index 9be45d2..bb640b4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.physical.impl.partitionsender;
import java.io.IOException;
-import java.util.concurrent.ExecutionException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
@@ -31,24 +30,24 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
-import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.ops.SenderStats;
import org.apache.drill.exec.physical.config.HashPartitionSender;
import org.apache.drill.exec.physical.impl.BaseRootExec;
-import org.apache.drill.exec.physical.impl.RootExec;
import org.apache.drill.exec.physical.impl.SendingAccountor;
-import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch;
-import org.apache.drill.exec.physical.impl.partitionsender.PartitionerTemplate.OutgoingRecordBatch;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.record.*;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.FragmentWritableBatch;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.rpc.data.DataTunnel;
+import org.apache.drill.exec.vector.CopyUtil;
import com.sun.codemodel.JExpr;
import com.sun.codemodel.JExpression;
import com.sun.codemodel.JType;
-import org.apache.drill.exec.vector.CopyUtil;
public class PartitionSenderRootExec extends BaseRootExec {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
index 1e6e71b..9bb24d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerTemplate.java
@@ -17,11 +17,12 @@
*/
package org.apache.drill.exec.physical.impl.partitionsender;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
import javax.inject.Named;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import io.netty.buffer.ByteBuf;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.compile.sig.RuntimeOverridden;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -35,22 +36,24 @@ import org.apache.drill.exec.physical.impl.SendingAccountor;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.FragmentWritableBatch;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.SchemaBuilder;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
-import org.apache.drill.exec.rpc.BaseRpcOutcomeListener;
-import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.data.DataTunnel;
-import org.apache.drill.exec.util.BatchPrinter;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
-import org.apache.drill.exec.work.ErrorHelper;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
public abstract class PartitionerTemplate implements Partitioner {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PartitionerTemplate.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
index 549bdce..9cb6f79 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/RecordBatchData.java
@@ -19,8 +19,12 @@ package org.apache.drill.exec.physical.impl.sort;
import java.util.List;
-import org.apache.drill.exec.record.*;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.vector.ValueVector;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
index 77582c3..558ce63 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
@@ -22,25 +22,29 @@ import java.util.List;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.SelectionVectorRemover;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractSingleRecordBatch;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
+import org.apache.drill.exec.vector.CopyUtil;
+import org.apache.drill.exec.vector.FixedWidthVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.allocator.FixedVectorAllocator;
import org.apache.drill.exec.vector.allocator.VariableEstimatedVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JExpression;
-import com.sun.codemodel.JVar;
public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVectorRemover>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemovingRecordBatch.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
index e17c06c..ddee38a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
@@ -17,17 +17,22 @@
*/
package org.apache.drill.exec.physical.impl.union;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.UnionAll;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import com.google.common.collect.Lists;
public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/BatchGroup.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/BatchGroup.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/BatchGroup.java
index 2be6871..5c2ab22 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/BatchGroup.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/BatchGroup.java
@@ -17,21 +17,26 @@
*/
package org.apache.drill.exec.physical.impl.xsort;
-import com.google.common.base.Stopwatch;
+import java.io.IOException;
+import java.util.Iterator;
+
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.cache.VectorAccessibleSerializable;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.vector.BigIntVector;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.concurrent.TimeUnit;
+import com.google.common.base.Stopwatch;
public class BatchGroup implements VectorAccessible {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BatchGroup.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 5582742..02b9ba0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -17,14 +17,16 @@
*/
package org.apache.drill.exec.physical.impl.xsort;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.sun.codemodel.JConditional;
-import com.sun.codemodel.JExpr;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
@@ -43,9 +45,15 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.ExternalSort;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.physical.impl.sort.SortRecordBatchBuilder;
-import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch;
-import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.util.Utilities;
@@ -56,9 +64,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.eigenbase.rel.RelFieldCollation.Direction;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExternalSortBatch.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/PriorityQueueCopierTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/PriorityQueueCopierTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/PriorityQueueCopierTemplate.java
index 686cc48..a74f14c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/PriorityQueueCopierTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/PriorityQueueCopierTemplate.java
@@ -17,24 +17,18 @@
*/
package org.apache.drill.exec.physical.impl.xsort;
-import com.google.common.collect.Lists;
-import com.google.common.base.Stopwatch;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
+import java.util.List;
+
+import javax.inject.Named;
+
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
-import org.apache.drill.exec.record.*;
-import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
-import javax.inject.Named;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
public abstract class PriorityQueueCopierTemplate implements PriorityQueueCopier {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PriorityQueueCopierTemplate.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
index ef9146a..add29c1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
@@ -22,8 +22,12 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.exception.FragmentSetupException;
-import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.physical.base.AbstractPhysicalVisitor;
+import org.apache.drill.exec.physical.base.Exchange;
import org.apache.drill.exec.physical.base.GroupScan;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.Store;
+import org.apache.drill.exec.physical.base.SubScan;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
index 6d720a7..096ac08 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/StatsCollector.java
@@ -17,8 +17,12 @@
*/
package org.apache.drill.exec.planner.fragment;
-import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.physical.base.Exchange;
import org.apache.drill.exec.physical.base.GroupScan;
+import org.apache.drill.exec.physical.base.HasAffinity;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.Store;
+import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.planner.AbstractOpWrapperVisitor;
import org.apache.drill.exec.planner.fragment.Fragment.ExchangeFragmentPair;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
index c966a84..66fc563 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
@@ -17,14 +17,17 @@
*/
package org.apache.drill.exec.planner.logical;
+import java.util.logging.Logger;
+
import org.eigenbase.rel.AggregateRel;
import org.eigenbase.rel.InvalidRelException;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
import org.eigenbase.trace.EigenbaseTrace;
-import java.util.logging.Logger;
-
/**
* Rule that converts an {@link AggregateRel} to a {@link DrillAggregateRel}, implemented by a Drill "segment" operation
* followed by a "collapseaggregate" operation.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
index 0f63140..b861897 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
@@ -17,10 +17,17 @@
*/
package org.apache.drill.exec.planner.logical;
+import java.util.List;
+import java.util.logging.Logger;
+
import org.eigenbase.rel.InvalidRelException;
import org.eigenbase.rel.JoinRel;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelOptUtil;
+import org.eigenbase.relopt.RelTraitSet;
import org.eigenbase.reltype.RelDataTypeField;
import org.eigenbase.rex.RexBuilder;
import org.eigenbase.rex.RexNode;
@@ -30,10 +37,6 @@ import org.eigenbase.trace.EigenbaseTrace;
import com.google.common.collect.Lists;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Logger;
-
/**
* Rule that converts a {@link JoinRel} to a {@link DrillJoinRel}, which is implemented by Drill "join" operation.
*/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
index 9a256bb..d0cd832 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
@@ -24,7 +24,22 @@ import java.util.List;
import net.hydromatic.optiq.tools.RuleSet;
import org.apache.drill.exec.ops.QueryContext;
-import org.apache.drill.exec.planner.physical.*;
+import org.apache.drill.exec.planner.physical.ConvertCountToDirectScan;
+import org.apache.drill.exec.planner.physical.FilterPrule;
+import org.apache.drill.exec.planner.physical.HashAggPrule;
+import org.apache.drill.exec.planner.physical.HashJoinPrule;
+import org.apache.drill.exec.planner.physical.LimitPrule;
+import org.apache.drill.exec.planner.physical.MergeJoinPrule;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.planner.physical.ProjectPrule;
+import org.apache.drill.exec.planner.physical.PushLimitToTopN;
+import org.apache.drill.exec.planner.physical.ScanPrule;
+import org.apache.drill.exec.planner.physical.ScreenPrule;
+import org.apache.drill.exec.planner.physical.SortConvertPrule;
+import org.apache.drill.exec.planner.physical.SortPrule;
+import org.apache.drill.exec.planner.physical.StreamAggPrule;
+import org.apache.drill.exec.planner.physical.UnionAllPrule;
+import org.apache.drill.exec.planner.physical.WriterPrule;
import org.eigenbase.rel.RelFactories;
import org.eigenbase.rel.rules.MergeProjectRule;
import org.eigenbase.rel.rules.PushFilterPastJoinRule;
@@ -32,13 +47,11 @@ import org.eigenbase.rel.rules.PushFilterPastProjectRule;
import org.eigenbase.rel.rules.PushJoinThroughJoinRule;
import org.eigenbase.rel.rules.PushProjectPastFilterRule;
import org.eigenbase.rel.rules.PushProjectPastJoinRule;
-import org.eigenbase.rel.rules.PushSortPastProjectRule;
import org.eigenbase.rel.rules.ReduceAggregatesRule;
import org.eigenbase.rel.rules.RemoveDistinctAggregateRule;
import org.eigenbase.rel.rules.RemoveDistinctRule;
import org.eigenbase.rel.rules.RemoveSortRule;
import org.eigenbase.rel.rules.RemoveTrivialProjectRule;
-import org.eigenbase.rel.rules.SwapJoinRule;
import org.eigenbase.relopt.RelOptRule;
import org.eigenbase.relopt.volcano.AbstractConverter.ExpandConversionRule;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5da52cb5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
index 5c982e4..f968dbb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
@@ -17,9 +17,12 @@
*/
package org.apache.drill.exec.planner.logical;
-import org.eigenbase.rel.SortRel;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
+import org.eigenbase.rel.SortRel;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
/**
* Rule that converts an {@link SortRel} to a {@link DrillSortRel}, implemented by a Drill "order" operation.
[09/32] git commit: DRILL-935: Run-time code generation support for
function which decodes string/varbinary into complex JSON object.
Posted by ja...@apache.org.
DRILL-935: Run-time code generation support for function which decodes string/varbinary into complex JSON object.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c789f1c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c789f1c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c789f1c8
Branch: refs/heads/master
Commit: c789f1c8ace3e12d81ccae05e79e4539cf9c829c
Parents: aaaca6a
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Mon Jun 9 13:02:52 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 07:58:08 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/common/types/Types.java | 2 +-
.../src/main/codegen/templates/BaseWriter.java | 4 +
.../src/main/codegen/templates/ListWriters.java | 3 +-
.../exec/client/PrintingResultsListener.java | 1 +
.../drill/exec/expr/DrillFuncHolderExpr.java | 7 +-
.../exec/expr/ExpressionTreeMaterializer.java | 17 ++-
.../expr/fn/DrillComplexWriterFuncHolder.java | 107 ++++++++++++++++++
.../drill/exec/expr/fn/DrillFuncHolder.java | 16 ++-
.../drill/exec/expr/fn/FunctionConverter.java | 30 +++--
.../exec/expr/fn/impl/conv/JsonConvertFrom.java | 101 +++++++++++++++++
.../drill/exec/physical/impl/ScanBatch.java | 5 +
.../exec/physical/impl/TopN/TopNBatch.java | 6 +
.../exec/physical/impl/WireRecordBatch.java | 5 +
.../impl/project/ProjectRecordBatch.java | 105 ++++++++++++-----
.../IteratorValidatorBatchIterator.java | 6 +
.../drill/exec/record/AbstractRecordBatch.java | 9 +-
.../apache/drill/exec/record/RecordBatch.java | 3 +
.../drill/exec/record/VectorContainer.java | 23 +++-
.../exec/store/easy/json/JSONRecordReader2.java | 5 +-
.../exec/vector/complex/fn/JsonReader.java | 30 +----
.../vector/complex/fn/JsonReaderWithState.java | 62 ++++++++++
.../vector/complex/impl/ComplexWriterImpl.java | 30 ++++-
.../complex/writer/TestComplexTypeWriter.java | 113 +++++++++++++++++++
.../vector/complex/writer/TestJsonReader.java | 3 +-
24 files changed, 614 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/common/src/main/java/org/apache/drill/common/types/Types.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java
index 10d84e1..9280d03 100644
--- a/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -349,7 +349,7 @@ public class Types {
case "binary":
return withMode(MinorType.VARBINARY, mode);
case "json":
- return withMode(MinorType.VARBINARY, mode);
+ return withMode(MinorType.LATE, mode);
default:
throw new UnsupportedOperationException("Could not determine type: " + typeName);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/codegen/templates/BaseWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/BaseWriter.java b/exec/java-exec/src/main/codegen/templates/BaseWriter.java
index 0c241b2..e281bc7 100644
--- a/exec/java-exec/src/main/codegen/templates/BaseWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/BaseWriter.java
@@ -74,6 +74,10 @@ public interface BaseWriter extends Positionable{
MapWriter rootAsMap();
ListWriter rootAsList();
boolean ok();
+
+ public void setPosition(int index);
+ public void setValueCount(int count);
+ public void reset();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/codegen/templates/ListWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ListWriters.java b/exec/java-exec/src/main/codegen/templates/ListWriters.java
index 64d92e2..278fddc 100644
--- a/exec/java-exec/src/main/codegen/templates/ListWriters.java
+++ b/exec/java-exec/src/main/codegen/templates/ListWriters.java
@@ -121,7 +121,8 @@ public class ${mode}ListWriter extends AbstractFieldWriter{
switch(mode){
case INIT:
int vectorCount = container.size();
- Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
+ Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
+ innerVector = vector;
writer = new Repeated${capName}WriterImpl(vector, this);
if(vectorCount != container.size()) writer.allocate();
writer.setPosition(${index});
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
index 4a18149..a5b65ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
@@ -86,6 +86,7 @@ public class PrintingResultsListener implements UserResultsListener {
if (isLastChunk) {
allocator.close();
latch.countDown();
+ System.out.println("Total rows returned : " + count.get());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
index e6370d8..f164bd8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
@@ -22,12 +22,13 @@ import java.util.List;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.drill.common.types.TypeProtos.MajorType;
-
+import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
public class DrillFuncHolderExpr extends FunctionHolderExpression implements Iterable<LogicalExpression>{
@@ -66,4 +67,8 @@ public class DrillFuncHolderExpr extends FunctionHolderExpression implements Ite
public boolean argConstantOnly(int i) {
return holder.isConstant(i);
}
+
+ public boolean isComplexWriterFuncHolder() {
+ return holder instanceof DrillComplexWriterFuncHolder;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 99d423f..9b80dc0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -25,6 +25,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ConvertExpression;
import org.apache.drill.common.expression.ErrorCollector;
@@ -63,6 +64,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.fn.HiveFuncHolder;
@@ -81,7 +83,12 @@ public class ExpressionTreeMaterializer {
};
public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry) {
- LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector), registry);
+ return ExpressionTreeMaterializer.materialize(expr, batch, errorCollector, registry, false);
+ }
+
+ public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry,
+ boolean allowComplexWriterExpr) {
+ LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr), registry);
if(out instanceof NullExpression){
return new TypedNullConstant(Types.optional(MinorType.INT));
}else{
@@ -93,10 +100,12 @@ public class ExpressionTreeMaterializer {
private ExpressionValidator validator = new ExpressionValidator();
private final ErrorCollector errorCollector;
private final VectorAccessible batch;
+ private final boolean allowComplexWriter;
- public MaterializeVisitor(VectorAccessible batch, ErrorCollector errorCollector) {
+ public MaterializeVisitor(VectorAccessible batch, ErrorCollector errorCollector, boolean allowComplexWriter) {
this.batch = batch;
this.errorCollector = errorCollector;
+ this.allowComplexWriter = allowComplexWriter;
}
private LogicalExpression validateNewExpr(LogicalExpression newExpr) {
@@ -135,6 +144,10 @@ public class ExpressionTreeMaterializer {
DrillFuncHolder matchedFuncHolder =
resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(call.getName()), call);
+ if (matchedFuncHolder instanceof DrillComplexWriterFuncHolder && ! allowComplexWriter) {
+ errorCollector.addGeneralError(call.getPosition(), "Only ProjectRecordBatch could have complex writer function. You are using complex writer function " + call.getName() + " in a non-project operation!");
+ }
+
//new arg lists, possible with implicit cast inserted.
List<LogicalExpression> argsWithCast = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
new file mode 100644
index 0000000..0efee63
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.fn;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.physical.impl.project.ProjectRecordBatch;
+import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
+import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JMod;
+import com.sun.codemodel.JVar;
+
+public class DrillComplexWriterFuncHolder extends DrillSimpleFuncHolder{
+
+ private FieldReference ref;
+
+ public DrillComplexWriterFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
+ String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
+ Map<String, String> methods, List<String> imports) {
+ super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports);
+ }
+
+ public void setReference(FieldReference ref) {
+ this.ref = ref;
+ }
+
+ @Override
+ protected HoldingContainer generateEvalBody(ClassGenerator<?> g, HoldingContainer[] inputVariables, String body, JVar[] workspaceJVars) {
+
+ g.getEvalBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", registeredNames[0]));
+
+ JBlock sub = new JBlock(true, true);
+ JBlock topSub = sub;
+
+ JVar complexWriter = g.declareClassField("complexWriter", g.getModel()._ref(ComplexWriter.class));
+
+ JClass cwClass = g.getModel().ref(ComplexWriterImpl.class);
+
+ JInvocation container = g.getMappingSet().getOutgoing().invoke("getOutgoingContainer");
+
+ //Default name is "col", if not passed in a reference name for the output vector.
+ String refName = ref == null? "col" : ref.getRootSegment().getPath();
+
+ g.getSetupBlock().assign(complexWriter, cwClass.staticInvoke("getWriter").arg(refName).arg(container));
+
+ JClass projBatchClass = g.getModel().ref(ProjectRecordBatch.class);
+ JExpression projBatch = JExpr.cast(projBatchClass, g.getMappingSet().getOutgoing());
+
+ g.getSetupBlock().add(projBatch.invoke("addComplexWriter").arg(complexWriter));
+
+
+ g.getEvalBlock().add(complexWriter.invoke("setPosition").arg(g.getMappingSet().getValueWriteIndex()));
+
+ sub.decl(g.getModel()._ref(ComplexWriter.class), returnValue.name, complexWriter);
+
+ // add the subblock after the out declaration.
+ g.getEvalBlock().add(topSub);
+
+ addProtectedBlock(g, sub, body, inputVariables, workspaceJVars, false);
+
+
+ JConditional jc = g.getEvalBlock()._if(complexWriter.invoke("ok").not());
+
+ jc._then().add(complexWriter.invoke("reset"));
+ //jc._then().directStatement("System.out.println(\"debug : write ok fail!, inIndex = \" + inIndex);");
+ jc._then()._return(JExpr.FALSE);
+
+ //jc._else().directStatement("System.out.println(\"debug : write successful, inIndex = \" + inIndex);");
+
+ g.getEvalBlock().directStatement(String.format("//---- end of eval portion of %s function. ----//", registeredNames[0]));
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index 5d5acf0..bbe76de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -236,8 +236,9 @@ public abstract class DrillFuncHolder {
public static class ValueReference {
MajorType type;
String name;
- boolean isConstant;
- boolean isFieldReader;
+ boolean isConstant = false;
+ boolean isFieldReader = false;
+ boolean isComplexWriter = false;
public ValueReference(MajorType type, String name) {
super();
@@ -245,8 +246,6 @@ public abstract class DrillFuncHolder {
Preconditions.checkNotNull(name);
this.type = type;
this.name = name;
- isConstant = false;
- this.isFieldReader = false;
}
public void setConstant(boolean isConstant) {
@@ -265,6 +264,15 @@ public abstract class DrillFuncHolder {
return ref;
}
+
+ public static ValueReference createComplexWriterRef(String name) {
+ MajorType type = Types.required(MinorType.LATE);
+ ValueReference ref = new ValueReference(type, name);
+ ref.isComplexWriter = true;
+
+ return ref;
+ }
+
}
public static class WorkspaceReference {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 44dd49d..75c3cbb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -37,6 +37,7 @@ import org.apache.drill.exec.expr.fn.DrillFuncHolder.ValueReference;
import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference;
import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import org.codehaus.commons.compiler.CompileException;
import org.codehaus.janino.Java.CompilationUnit;
import org.codehaus.janino.Parser;
@@ -129,6 +130,17 @@ public class FunctionConverter {
continue;
}
+ // Special processing for @Output ComplexWriter
+ if (output != null && ComplexWriter.class.isAssignableFrom(field.getType())) {
+
+ if(outputField != null){
+ return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", clazz, field);
+ }else{
+ outputField = ValueReference.createComplexWriterRef(field.getName());
+ }
+ continue;
+ }
+
// check that param and output are value holders.
if(!ValueHolder.class.isAssignableFrom(field.getType())){
return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder interface. All fields of type @Param or @Output must extend this interface..", field.getType()), clazz, field);
@@ -154,15 +166,11 @@ public class FunctionConverter {
return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", clazz, field);
}else{
outputField = p;
-
}
-
}
-
+
}else{
// workspace work.
-// logger.debug("Found workspace field {}:{}", field.getType(), field.getName());
- //workspaceFields.add(new WorkspaceReference(field.getType(), field.getName()));
WorkspaceReference wsReference = new WorkspaceReference(field.getType(), field.getName());
if (template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(field.getType()) ) {
@@ -217,8 +225,16 @@ public class FunctionConverter {
return new DrillDecimalAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
case SIMPLE:
- return new DrillSimpleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
- template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
+ if (outputField.isComplexWriter)
+ return new DrillComplexWriterFuncHolder(template.scope(), template.nulls(),
+ template.isBinaryCommutative(),
+ template.isRandom(), registeredNames,
+ ps, outputField, works, methods, imports);
+ else
+ return new DrillSimpleFuncHolder(template.scope(), template.nulls(),
+ template.isBinaryCommutative(),
+ template.isRandom(), registeredNames,
+ ps, outputField, works, methods, imports);
case DECIMAL_MAX_SCALE:
return new DrillDecimalMaxScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
new file mode 100644
index 0000000..d8df543
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.fn.impl.conv;
+
+
+import java.io.ByteArrayOutputStream;
+import java.io.StringReader;
+
+import io.netty.buffer.ByteBuf;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.complex.fn.JsonWriter;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
+import com.google.common.base.Charsets;
+
+public class JsonConvertFrom {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonConvertFrom.class);
+
+ private JsonConvertFrom(){}
+
+ @FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL, isRandom = true)
+ public static class ConvertFromJson implements DrillSimpleFunc{
+
+ @Param VarBinaryHolder in;
+ @Output ComplexWriter writer;
+
+ public void setup(RecordBatch incoming){
+ }
+
+ public void eval(){
+
+ byte[] buf = new byte[in.end - in.start];
+ in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
+
+ try {
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader();
+
+ jsonReader.write(new java.io.StringReader(input), writer);
+
+ } catch (Exception e) {
+ System.out.println(" msg = " + e.getMessage() + " trace : " + e.getStackTrace());
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL, isRandom = true)
+ public static class ConvertFromJsonVarchar implements DrillSimpleFunc{
+
+ @Param VarCharHolder in;
+ @Output ComplexWriter writer;
+
+ public void setup(RecordBatch incoming){
+ }
+
+ public void eval(){
+
+ byte[] buf = new byte[in.end - in.start];
+ in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
+
+ try {
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader();
+
+ jsonReader.write(new java.io.StringReader(input), writer);
+
+ } catch (Exception e) {
+ System.out.println(" msg = " + e.getMessage() + " trace : " + e.getStackTrace());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 2914b67..d142ff8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -284,4 +284,9 @@ public class ScanBatch implements RecordBatch {
oContext.close();
}
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 5196279..c9cd2dd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -363,6 +363,12 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
public Iterator<VectorWrapper<?>> iterator() {
return container.iterator();
}
+
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index fb681ff..c7fc813 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -147,4 +147,9 @@ public class WireRecordBatch implements RecordBatch {
fragProvider.cleanup();
}
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 61c256b..f5a4444 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -37,10 +37,12 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.DrillFuncHolderExpr;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
+import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Project;
@@ -50,8 +52,10 @@ import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.google.common.base.Preconditions;
@@ -63,6 +67,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
private Projector projector;
private List<ValueVector> allocationVectors;
+ private List<ComplexWriter> complexWriters;
private boolean hasRemainder = false;
private int remainderIndex = 0;
private int recordCount;
@@ -85,55 +90,47 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
return super.innerNext();
}
+ public VectorContainer getOutgoingContainer() {
+ return this.container;
+ }
+
@Override
protected void doWork() {
// VectorUtil.showVectorAccessibleContent(incoming, ",");
int incomingRecordCount = incoming.getRecordCount();
- for(ValueVector v : this.allocationVectors){
-// AllocationHelper.allocate(v, incomingRecordCount, 250);
-// v.allocateNew();
- v.allocateNewSafe();
- }
+
+ doAlloc();
+
int outputRecords = projector.projectRecords(0, incomingRecordCount, 0);
if (outputRecords < incomingRecordCount) {
- for(ValueVector v : allocationVectors){
- ValueVector.Mutator m = v.getMutator();
- m.setValueCount(outputRecords);
- }
+ setValueCount(outputRecords);
hasRemainder = true;
remainderIndex = outputRecords;
this.recordCount = remainderIndex;
} else {
- for(ValueVector v : allocationVectors){
- ValueVector.Mutator m = v.getMutator();
- m.setValueCount(incomingRecordCount);
- }
+ setValueCount(incomingRecordCount);
for(VectorWrapper<?> v: incoming) {
v.clear();
}
this.recordCount = outputRecords;
}
+ // In case of complex writer expression, vectors would be added to batch run-time.
+ // We have to re-build the schema.
+ if (complexWriters != null) {
+ container.buildSchema(SelectionVectorMode.NONE);
+ }
}
private void handleRemainder() {
int remainingRecordCount = incoming.getRecordCount() - remainderIndex;
- for(ValueVector v : this.allocationVectors){
- //AllocationHelper.allocate(v, remainingRecordCount, 250);
- v.allocateNewSafe();
- }
+ doAlloc();
int projRecords = projector.projectRecords(remainderIndex, remainingRecordCount, 0);
if (projRecords < remainingRecordCount) {
- for(ValueVector v : allocationVectors){
- ValueVector.Mutator m = v.getMutator();
- m.setValueCount(projRecords);
- }
+ setValueCount(projRecords);
this.recordCount = projRecords;
remainderIndex += projRecords;
} else {
- for(ValueVector v : allocationVectors){
- ValueVector.Mutator m = v.getMutator();
- m.setValueCount(remainingRecordCount);
- }
+ setValueCount(remainingRecordCount);
hasRemainder = false;
remainderIndex = 0;
for(VectorWrapper<?> v: incoming) {
@@ -141,8 +138,48 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
}
this.recordCount = remainingRecordCount;
}
+ // In case of complex writer expression, vectors would be added to batch run-time.
+ // We have to re-build the schema.
+ if (complexWriters != null) {
+ container.buildSchema(SelectionVectorMode.NONE);
+ }
}
+ public void addComplexWriter(ComplexWriter writer) {
+ complexWriters.add(writer);
+ }
+
+ private boolean doAlloc() {
+ //Allocate vv in the allocationVectors.
+ for(ValueVector v : this.allocationVectors){
+ //AllocationHelper.allocate(v, remainingRecordCount, 250);
+ if (!v.allocateNewSafe())
+ return false;
+ }
+
+ //Allocate vv for complexWriters.
+ if (complexWriters == null)
+ return true;
+
+ for (ComplexWriter writer : complexWriters)
+ writer.allocate();
+
+ return true;
+ }
+
+ private void setValueCount(int count) {
+ for(ValueVector v : allocationVectors){
+ ValueVector.Mutator m = v.getMutator();
+ m.setValueCount(count);
+ }
+
+ if (complexWriters == null)
+ return;
+
+ for (ComplexWriter writer : complexWriters)
+ writer.setValueCount(count);
+ }
+
/** hack to make ref and full work together... need to figure out if this is still necessary. **/
private FieldReference getRef(NamedExpression e){
FieldReference ref = e.getRef();
@@ -200,13 +237,12 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
}else{
for(int i = 0; i < exprs.size(); i++){
final NamedExpression namedExpression = exprs.get(i);
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry());
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
final MaterializedField outputField = MaterializedField.create(getRef(namedExpression), expr.getMajorType());
if(collector.hasErrors()){
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
}
-
// add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
if(expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE
&& !((ValueVectorReadExpression) expr).hasReadPath()
@@ -223,7 +259,17 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
container.add(tp.getTo());
transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
// logger.debug("Added transfer.");
- }else{
+ } else if (expr instanceof DrillFuncHolderExpr &&
+ ((DrillFuncHolderExpr) expr).isComplexWriterFuncHolder()) {
+ // Need to process ComplexWriter function evaluation.
+ // Lazy initialization of the list of complex writers, if not done yet.
+ if (complexWriters == null)
+ complexWriters = Lists.newArrayList();
+
+ // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
+ ((DrillComplexWriterFuncHolder) ((DrillFuncHolderExpr) expr).getHolder()).setReference(namedExpression.getRef());
+ cg.addExpr(expr);
+ } else{
// need to do evaluation.
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
allocationVectors.add(vector);
@@ -231,13 +277,10 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
HoldingContainer hc = cg.addExpr(write);
-
cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
logger.debug("Added eval.");
}
}
-
-
}
cg.rotateBlock();
cg.getEvalBlock()._return(JExpr.TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
index 1f73192..19f6497 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
@@ -24,6 +24,7 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
@@ -137,4 +138,9 @@ public class IteratorValidatorBatchIterator implements RecordBatch {
public void cleanup() {
incoming.cleanup();
}
+
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index 7b7b708..d71b811 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -31,7 +31,7 @@ import org.apache.drill.exec.record.selection.SelectionVector4;
public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements RecordBatch{
final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
- protected final VectorContainer container = new VectorContainer();
+ protected final VectorContainer container; //= new VectorContainer();
protected final T popConfig;
protected final FragmentContext context;
protected final OperatorContext oContext;
@@ -43,6 +43,7 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
this.popConfig = popConfig;
this.oContext = new OperatorContext(popConfig, context);
this.stats = oContext.getStats();
+ this.container = new VectorContainer(this.oContext);
}
@Override
@@ -135,4 +136,10 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
return batch;
}
+
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
index 75b9b0c..7617d91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
@@ -85,6 +85,9 @@ public interface RecordBatch extends VectorAccessible {
public abstract SelectionVector4 getSelectionVector4();
+
+ public VectorContainer getOutgoingContainer();
+
/**
* Get the value vector type and id for the given schema path. The TypedFieldId should store a fieldId which is the
* same as the ordinal position of the field within the Iterator provided this classes implementation of
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index 3c67466..1f2c33a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -26,6 +26,8 @@ import java.util.List;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractMapVector;
@@ -39,8 +41,14 @@ public class VectorContainer extends AbstractMapVector implements Iterable<Vecto
protected final List<VectorWrapper<?>> wrappers = Lists.newArrayList();
private BatchSchema schema;
private int recordCount = -1;
-
+ private final OperatorContext oContext;
+
public VectorContainer() {
+ this.oContext = null;
+ }
+
+ public VectorContainer( OperatorContext oContext) {
+ this.oContext = oContext;
}
// public VectorContainer(List<ValueVector> vectors, List<ValueVector[]> hyperVectors) {
@@ -66,8 +74,17 @@ public class VectorContainer extends AbstractMapVector implements Iterable<Vecto
add(vv, releasable);
}
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz){
- return null;
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz){
+ MaterializedField field = MaterializedField.create(name, type);
+ ValueVector v = TypeHelper.getNewVector(field, this.oContext.getAllocator());
+
+ add(v);
+
+ if(clazz.isAssignableFrom(v.getClass())){
+ return (T) v;
+ }else{
+ throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
index d316830..028a7d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
@@ -30,6 +30,7 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.vector.complex.fn.JsonReader;
+import org.apache.drill.exec.vector.complex.fn.JsonReaderWithState;
import org.apache.drill.exec.vector.complex.fn.JsonRecordSplitter;
import org.apache.drill.exec.vector.complex.fn.UTF8JsonRecordSplitter;
import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
@@ -46,7 +47,7 @@ public class JSONRecordReader2 implements RecordReader{
private Path hadoopPath;
private FileSystem fileSystem;
private InputStream stream;
- private JsonReader jsonReader;
+ private JsonReaderWithState jsonReader;
public JSONRecordReader2(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem,
List<SchemaPath> columns) throws OutOfMemoryException {
@@ -61,7 +62,7 @@ public class JSONRecordReader2 implements RecordReader{
JsonRecordSplitter splitter = new UTF8JsonRecordSplitter(stream);
this.writer = new VectorContainerWriter(output);
this.mutator = output;
- jsonReader = new JsonReader(splitter);
+ jsonReader = new JsonReaderWithState(splitter);
}catch(IOException e){
throw new ExecutionSetupException("Failure reading JSON file.", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index e5d4b5b..485d134 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -43,30 +43,16 @@ import com.google.common.base.Charsets;
public class JsonReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonReader.class);
- public static enum WriteState {
- WRITE_SUCCEED, WRITE_FAILED, NO_MORE
- }
private final JsonFactory factory = new JsonFactory();
- private ByteBufInputStream stream;
- private long byteOffset;
- private JsonRecordSplitter splitter;
- private Reader reader;
private JsonParser parser;
- public JsonReader(JsonRecordSplitter splitter) throws JsonParseException, IOException {
- this.splitter = splitter;
+ public JsonReader() throws JsonParseException, IOException {
factory.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
factory.configure(Feature.ALLOW_COMMENTS, true);
- reader = splitter.getNextReader();
}
- public WriteState write(ComplexWriter writer) throws JsonParseException, IOException {
- if(reader == null){
- reader = splitter.getNextReader();
- if(reader == null) return WriteState.NO_MORE;
-
- }
+ public boolean write(Reader reader, ComplexWriter writer) throws JsonParseException, IOException {
parser = factory.createJsonParser(reader);
reader.mark(1024*128);
@@ -82,7 +68,7 @@ public class JsonReader {
writeData(writer.rootAsList());
break;
case NOT_AVAILABLE:
- return WriteState.NO_MORE;
+ return false;
default:
throw new JsonParseException(
String.format("Failure while parsing JSON. Found token of [%s] Drill currently only supports parsing "
@@ -91,13 +77,7 @@ public class JsonReader {
parser.getCurrentLocation());
}
- if (!writer.ok()) {
- reader.reset();
- return WriteState.WRITE_FAILED;
- } else {
- reader = null;
- return WriteState.WRITE_SUCCEED;
- }
+ return true;
}
@@ -227,5 +207,7 @@ public class JsonReader {
}
}
list.end();
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
new file mode 100644
index 0000000..cf885a4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.vector.complex.fn;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
+import com.fasterxml.jackson.core.JsonParseException;
+
+public class JsonReaderWithState {
+
+ public static enum WriteState {
+ WRITE_SUCCEED, WRITE_FAILED, NO_MORE
+ }
+
+ private Reader reader;
+ private JsonRecordSplitter splitter;
+ private JsonReader jsonReader;
+
+ public JsonReaderWithState(JsonRecordSplitter splitter) throws IOException{
+ this.splitter = splitter;
+ reader = splitter.getNextReader();
+ jsonReader = new JsonReader();
+ }
+
+ public WriteState write(ComplexWriter writer) throws JsonParseException, IOException {
+ if (reader == null) {
+ reader = splitter.getNextReader();
+ if (reader == null)
+ return WriteState.NO_MORE;
+
+ }
+
+ jsonReader.write(reader, writer);
+
+ if (!writer.ok()) {
+ reader.reset();
+ return WriteState.WRITE_FAILED;
+ } else {
+ reader = null;
+ return WriteState.WRITE_SUCCEED;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
index 0c7378a..2fa72f7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
@@ -17,11 +17,15 @@
*/
package org.apache.drill.exec.vector.complex.impl;
+import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.StateTool;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import com.google.common.base.Preconditions;
@@ -160,5 +164,29 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
return listRoot;
}
+
+ private static class VectorAccessibleFacade extends MapVector {
+
+ private final VectorContainer vc;
+
+ public VectorAccessibleFacade(VectorContainer vc) {
+ super("", null);
+ this.vc = vc;
+ }
+
+ @Override
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ ValueVector v = vc.addOrGet(name, type, clazz);
+ this.put(name, v);
+ return this.typeify(v, clazz);
+
+ }
+
+ }
+
+ public static ComplexWriter getWriter(String name, VectorContainer container){
+ VectorAccessibleFacade vc = new VectorAccessibleFacade(container);
+ return new ComplexWriterImpl(name, vc);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
new file mode 100644
index 0000000..9011ff3
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.vector.complex.writer;
+
+import org.apache.drill.BaseTestQuery;
+import org.junit.Test;
+
+public class TestComplexTypeWriter extends BaseTestQuery{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class);
+
+ @Test
+ //basic case. convert varchar into json.
+ public void testA0() throws Exception{
+ test(" select convert_from('{x:100, y:215.6}' ,'JSON') as mycol from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //map contains int, float, repeated list , repeated map, nested repeated map, etc.
+ public void testA1() throws Exception{
+ test(" select convert_from('{x:100, y:215.6, z: [1, 2, 3], s : [[5, 6, 7], [8, 9]], " +
+ " t : [{a : 100, b: 200}, {a:300, b: 400}], " +
+ " nrmp: [ { x: [{ id: 123}], y: { y : \"SQL\"} }] }' ,'JSON') " +
+ " as mycol from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //two convert functions.
+ public void testA2() throws Exception{
+ test(" select convert_from('{x:100, y:215.6}' ,'JSON') as mycol1, convert_from('{x:100, y:215.6}' ,'JSON') as mycol2 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //two convert functions. One convert's input comes from a string concat function.
+ public void testA3() throws Exception{
+ test(" select convert_from(concat('{x:100,', 'y:215.6}') ,'JSON') as mycol1, convert_from('{x:100, y:215.6}' ,'JSON') as mycol2 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //two convert functions. One's input is an empty map.
+ public void testA4() throws Exception{
+ test(" select convert_from('{}' ,'JSON') as mycol1, convert_from('{x:100, y:215.6}' ,'JSON') as mycol2 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //two convert functions. One's input is an empty list ( ok to have null in the result?)
+ public void testA5() throws Exception{
+ test(" select convert_from('[]' ,'JSON') as mycol1, convert_from('{x:100, y:215.6}' ,'JSON') as mycol2 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //input is a list of BigInt. Output will be a repeated list vector.
+ public void testA6() throws Exception{
+ test(" select convert_from('[1, 2, 3]' ,'JSON') as mycol1 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //input is a list of float. Output will be a repeated list vector.
+ public void testA7() throws Exception{
+ test(" select convert_from('[1.2, 2.3, 3.5]' ,'JSON') as mycol1 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //input is a list of list of big int. Output will be a repeated list vector.
+ public void testA8() throws Exception{
+ test(" select convert_from('[ [1, 2], [3, 4], [5]]' ,'JSON') as mycol1 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //input is a list of map. Output will be a repeated list vector.
+ public void testA9() throws Exception{
+ test(" select convert_from('[{a : 100, b: 200}, {a:300, b: 400}]' ,'JSON') as mycol1 from cp.`tpch/nation.parquet`;");
+ }
+
+ @Test
+ //two convert functions, one regular nest functions, used with Filter op.
+ public void testA10() throws Exception{
+ test(" select convert_from('{x:100, y:215.6}' ,'JSON') as mycol1, " +
+ " convert_from('{x:200, y:678.9}' ,'JSON') as mycol2, " +
+ " 1 + 2 * 3 as numvalue " +
+ " from cp.`tpch/nation.parquet` where n_nationkey > 5;");
+ }
+
+ @Test
+ //convert from string constructed from columns in parquet file.
+ public void testA11() throws Exception{
+ test(" select convert_from(concat(concat('{ NationName: \"', N_NAME) , '\"}'), 'JSON')" +
+ " from cp.`tpch/nation.parquet` where n_nationkey > 5;");
+ }
+
+ @Test
+ //Test multiple batches creation ( require multiple alloc for complex writer during Project ).
+ public void testA100() throws Exception{
+ test(" select convert_from(concat(concat('{ Price : ', L_EXTENDEDPRICE) , '}') , 'JSON') " +
+ " from cp.`tpch/lineitem.parquet` limit 10; ");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c789f1c8/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 1f3b540..8b31958 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -26,6 +26,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.fn.JsonReader;
+import org.apache.drill.exec.vector.complex.fn.JsonReaderWithState;
import org.apache.drill.exec.vector.complex.fn.JsonWriter;
import org.apache.drill.exec.vector.complex.fn.ReaderJSONRecordSplitter;
import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
@@ -87,7 +88,7 @@ public class TestJsonReader {
writer.allocate();
- JsonReader jsonReader = new JsonReader(new ReaderJSONRecordSplitter(compound));
+ JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound));
int i =0;
List<Integer> batchSizes = Lists.newArrayList();
[27/32] git commit: DRILL-978: Consider output type when we cost
matching FunctionCall to FunctionHolder
Posted by ja...@apache.org.
DRILL-978: Consider output type when we cost matching FunctionCall to FunctionHolder
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9e7b14a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9e7b14a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9e7b14a5
Branch: refs/heads/master
Commit: 9e7b14a559e9361e7b6a43cd625571c0c3a72b2e
Parents: c8db346
Author: Mehant Baid <me...@gmail.com>
Authored: Thu Jun 12 18:09:57 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:28 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/expr/fn/DrillFuncHolder.java | 8 ++++++++
.../org/apache/drill/exec/resolver/TypeCastRules.java | 10 ++++++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9e7b14a5/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index bbe76de..fd687af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -293,4 +293,12 @@ public abstract class DrillFuncHolder {
}
}
+
+ public boolean matchInputOutputType() {
+ return false;
+ }
+
+ public MajorType getReturnType() {
+ return returnValue.type;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9e7b14a5/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index 5c5ff80..854342c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -794,6 +794,16 @@ public class TypeCastRules {
// number of arguments that could implicitly casts using precedence map or didn't require casting at all
int nCasts = 0;
+ // Check if the function holder requires the input type and output type to match
+ if (holder.matchInputOutputType() == true) {
+ MinorType outputType = holder.getReturnType(call.args).getMinorType();
+ for (int i = 0; i < holder.getParamCount(); i++) {
+ if (call.args.get(i).getMajorType().getMinorType() != outputType) {
+ return -1;
+ }
+ }
+ }
+
for (int i = 0; i < holder.getParamCount(); i++) {
MajorType argType = call.args.get(i).getMajorType();
MajorType parmType = holder.getParmMajorType(i);
[23/32] DRILL-933: Remove old physical operator cost & size concepts,
add automatic size-based parallelization
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index e69f61c..bc8495c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -31,11 +31,11 @@ import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
+import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS;
@@ -357,28 +357,11 @@ public class ParquetGroupScan extends AbstractGroupScan {
return columns;
}
- @Override
- public OperatorCost getCost() {
- // TODO Figure out how to properly calculate cost
- return new OperatorCost(1, rowGroupInfos.size(), 1, 1);
- }
@Override
- public Size getSize() {
-// long totalSize = 0;
-// for (RowGroupInfo rowGrpInfo : rowGroupInfos) {
-// totalSize += rowGrpInfo.getTotalBytes();
-// }
-// int rowSize = (int) (totalSize/rowCount);
-
- // if all the columns are required.
- if (columns == null || columns.isEmpty()) {
- return new Size(rowCount, columnValueCounts.size());
- } else {
- // project pushdown : subset of columns are required.
- return new Size(rowCount, columns.size());
- }
-
+ public ScanStats getScanStats() {
+ int columnCount = columns == null ? 20 : columns.size();
+ return new ScanStats(GroupScanProperty.EXACT_ROW_COUNT, rowCount, 1, rowCount * columnCount);
}
@Override
@@ -412,11 +395,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
return true;
}
- @Override
- public GroupScanProperty getProperty() {
- return GroupScanProperty.EXACT_ROW_COUNT;
- }
-
/**
* Return column value count for the specified column. If does not contain such column, return 0.
*/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index 1f66e9f..d1a086c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -22,15 +22,12 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
@@ -102,16 +99,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
}
@Override
- public OperatorCost getCost() {
- return null;
- }
-
- @Override
- public Size getSize() {
- return null;
- }
-
- @Override
public boolean isExecutable() {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
index 67bab5e..75f0e74 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetWriter.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
@@ -90,12 +89,6 @@ public class ParquetWriter extends AbstractWriter {
}
@Override
- public OperatorCost getCost() {
- // TODO:
- return new OperatorCost(1,1,1,1);
- }
-
- @Override
public int getOperatorType() {
return CoreOperatorType.PARQUET_WRITER_VALUE;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
index 09aabb0..d260927 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
@@ -25,11 +25,11 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.ScanStats;
+import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
@@ -62,14 +62,8 @@ public class SystemTableScan extends AbstractGroupScan implements SubScan{
this.plugin = plugin;
}
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1,1,1,1);
- }
-
- @Override
- public Size getSize() {
- return new Size(100,1);
+ public ScanStats getScanStats(){
+ return ScanStats.TRIVIAL_TABLE;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 94e2cd4..3048f77 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -22,14 +22,16 @@ import io.netty.buffer.ByteBuf;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.DistributedCache.CacheConfig;
import org.apache.drill.exec.cache.DistributedCache.SerializationMode;
+import org.apache.drill.exec.coord.DistributedSemaphore;
+import org.apache.drill.exec.coord.DistributedSemaphore.DistributedLease;
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.exception.OptimizerException;
import org.apache.drill.exec.ops.QueryContext;
@@ -74,6 +76,7 @@ import com.google.common.collect.Lists;
public class Foreman implements Runnable, Closeable, Comparable<Object>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Foreman.class);
+
public static final CacheConfig<FragmentHandle, PlanFragment> FRAGMENT_CACHE = CacheConfig //
.newBuilder(FragmentHandle.class, PlanFragment.class) //
.mode(SerializationMode.PROTOBUF) //
@@ -86,12 +89,33 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
private WorkerBee bee;
private UserClientConnection initiatingClient;
private final AtomicState<QueryState> state;
+ private final DistributedSemaphore smallSemaphore;
+ private final DistributedSemaphore largeSemaphore;
+ private final long queueThreshold;
+ private final long queueTimeout;
+ private volatile DistributedLease lease;
+ private final boolean queuingEnabled;
public Foreman(WorkerBee bee, DrillbitContext dContext, UserClientConnection connection, QueryId queryId,
RunQuery queryRequest) {
this.queryId = queryId;
this.queryRequest = queryRequest;
this.context = new QueryContext(connection.getSession(), queryId, dContext);
+ this.queuingEnabled = context.getOptions().getOption(ExecConstants.ENABLE_QUEUE_KEY).bool_val;
+ if(queuingEnabled){
+ int smallQueue = context.getOptions().getOption(ExecConstants.SMALL_QUEUE_KEY).num_val.intValue();
+ int largeQueue = context.getOptions().getOption(ExecConstants.LARGE_QUEUE_KEY).num_val.intValue();
+ this.largeSemaphore = dContext.getClusterCoordinator().getSemaphore("query.large", largeQueue);
+ this.smallSemaphore = dContext.getClusterCoordinator().getSemaphore("query.small", smallQueue);
+ this.queueThreshold = context.getOptions().getOption(ExecConstants.QUEUE_THRESHOLD_KEY).num_val;
+ this.queueTimeout = context.getOptions().getOption(ExecConstants.QUEUE_TIMEOUT_KEY).num_val;
+ }else{
+ this.largeSemaphore = null;
+ this.smallSemaphore = null;
+ this.queueThreshold = 0;
+ this.queueTimeout = 0;
+ }
+
this.initiatingClient = connection;
this.fragmentManager = new QueryManager(queryId, queryRequest, bee.getContext().getPersistentStoreProvider(), new ForemanManagerListener(), dContext.getController(), this);
this.bee = bee;
@@ -194,10 +218,21 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
System.out.flush();
System.exit(-1);
}finally{
+ releaseLease();
Thread.currentThread().setName(originalThread);
}
}
+ private void releaseLease(){
+ if(lease != null){
+ try{
+ lease.close();
+ }catch(Exception e){
+ logger.warn("Failure while releasing lease.", e);
+ };
+ }
+
+ }
private void parseAndRunLogicalPlan(String json) {
try {
@@ -260,7 +295,6 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
}
-
private void parseAndRunPhysicalPlan(String json) {
try {
PhysicalPlan plan = context.getPlanReader().readPhysicalPlan(json);
@@ -272,6 +306,9 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
private void runPhysicalPlan(PhysicalPlan plan) {
+
+
+
if(plan.getProperties().resultMode != ResultMode.EXEC){
fail(String.format("Failure running plan. You requested a result mode of %s and a physical plan can only be output as EXEC", plan.getProperties().resultMode), new Exception());
}
@@ -287,12 +324,20 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
- SimpleParallelizer parallelizer = new SimpleParallelizer()
- .setGlobalMaxWidth(context.getConfig().getInt(ExecConstants.GLOBAL_MAX_WIDTH))
- .setMaxWidthPerEndpoint(context.getConfig().getInt(ExecConstants.MAX_WIDTH_PER_ENDPOINT))
- .setAffinityFactor(context.getConfig().getDouble(ExecConstants.AFFINITY_FACTOR));
+ SimpleParallelizer parallelizer = new SimpleParallelizer(context);
try {
+
+ double size = 0;
+ for(PhysicalOperator ops : plan.getSortedOperators()){
+ size += ops.getCost();
+ }
+ if(queuingEnabled && size > this.queueThreshold){
+ this.lease = largeSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
+ }else{
+ this.lease = smallSemaphore.acquire(this.queueTimeout, TimeUnit.MILLISECONDS);
+ }
+
QueryWorkUnit work = parallelizer.getFragments(context.getOptions().getSessionOptionList(), context.getCurrentEndpoint(),
queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet);
@@ -324,7 +369,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
logger.debug("Fragments running.");
state.updateState(QueryState.PENDING, QueryState.RUNNING);
- } catch (ExecutionSetupException | RpcException e) {
+ } catch (Exception e) {
fail("Failure while setting up query.", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 562d7e8..7cfe51a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.TestTools;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.client.PrintingResultsListener;
@@ -99,8 +100,14 @@ public class BaseTestQuery extends ExecTest{
bit.run();
client = new DrillClient(config, serviceSet.getCoordinator());
client.connect();
+ List<QueryResultBatch> results = client.runQuery(QueryType.SQL, String.format("alter session set `%s` = 2", ExecConstants.MAX_WIDTH_PER_NODE_KEY));
+ for(QueryResultBatch b : results){
+ b.release();
+ }
}
+
+
protected BufferAllocator getAllocator(){
return allocator;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
index 1c5a7f9..b5b63bc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
@@ -25,7 +25,7 @@ public class TestTpchDistributed extends BaseTestQuery{
private void testDistributed(String fileName) throws Exception{
String query = getFile(fileName);
- test(query);
+ test("alter session set `planner.slice_target` = 10; " + query);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f54ab01/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java
index ea19351..f6972c3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java
@@ -50,7 +50,7 @@ public class TestFragmentChecker extends PopUnitTestBase{
PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance());
Fragment fragmentRoot = getRootFragment(ppr, fragmentFile);
PlanningSet planningSet = StatsCollector.collectStats(fragmentRoot);
- SimpleParallelizer par = new SimpleParallelizer();
+ SimpleParallelizer par = new SimpleParallelizer(1000*1000, 5, 10, 1.2);
List<DrillbitEndpoint> endpoints = Lists.newArrayList();
DrillbitEndpoint localBit = null;
for(int i =0; i < bitCount; i++){
@@ -59,7 +59,6 @@ public class TestFragmentChecker extends PopUnitTestBase{
endpoints.add(b1);
}
- par.setGlobalMaxWidth(10).setMaxWidthPerEndpoint(5);
QueryWorkUnit qwu = par.getFragments(new OptionList(), localBit, QueryId.getDefaultInstance(), endpoints, ppr, fragmentRoot, planningSet);
System.out.println(String.format("=========ROOT FRAGMENT [%d:%d] =========", qwu.getRootFragment().getHandle().getMajorFragmentId(), qwu.getRootFragment().getHandle().getMinorFragmentId()));
[21/32] git commit: DRILL-977: changed container style to fluid.
Added column filter.
Posted by ja...@apache.org.
DRILL-977: changed container style to fluid. Added column filter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/78bc8563
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/78bc8563
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/78bc8563
Branch: refs/heads/master
Commit: 78bc8563eb546f7c484b341b181a626294f8cfcd
Parents: 7821b51
Author: Sudheesh Katkam <sk...@maprtech.com>
Authored: Thu Jun 12 15:07:21 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:10 2014 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/resources/rest/generic.ftl | 4 ++--
.../src/main/resources/rest/query/result.ftl | 14 ++++++++++----
2 files changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/78bc8563/exec/java-exec/src/main/resources/rest/generic.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl
index 1a47915..c5ed400 100644
--- a/exec/java-exec/src/main/resources/rest/generic.ftl
+++ b/exec/java-exec/src/main/resources/rest/generic.ftl
@@ -41,7 +41,7 @@
<body role="document">
<div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
- <div class="container">
+ <div class="container-fluid">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="sr-only">Toggle navigation</span>
@@ -64,7 +64,7 @@
</div>
</div>
- <div class="container theme-showcase" role="main">
+ <div class="container-fluid" role="main">
<@page_body/>
</div>
</body>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/78bc8563/exec/java-exec/src/main/resources/rest/query/result.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/query/result.ftl b/exec/java-exec/src/main/resources/rest/query/result.ftl
index bedbf9f..a3cb0ce 100644
--- a/exec/java-exec/src/main/resources/rest/query/result.ftl
+++ b/exec/java-exec/src/main/resources/rest/query/result.ftl
@@ -17,6 +17,9 @@
<script type="text/javascript" language="javascript" src="//code.jquery.com/jquery-1.10.2.min.js"></script>
<script type="text/javascript" language="javascript" src="//cdn.datatables.net/1.10.0/js/jquery.dataTables.min.js"></script>
<script type="text/javascript" language="javascript" src="//cdn.datatables.net/plug-ins/be7019ee387/integration/jqueryui/dataTables.jqueryui.js"></script>
+
+ <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/colvis/1.1.0/css/dataTables.colVis.css">
+ <script type="text/javascript" language="javascript" src="//cdn.datatables.net/colvis/1.1.0/js/dataTables.colVis.min.js"></script>
</#macro>
<#macro page_body>
@@ -25,16 +28,16 @@
</div>
<h2>Result</h2>
<div style="width=100%; overflow: auto;">
- <table id="relation" class="table table-striped table-bordered table-condensed" style="display: table; table-layout: fized; width=100%;">
+ <table id="relation" class="table table-striped table-bordered table-condensed" style="table-layout: auto; width=100%;">
<#assign rows = model[0]>
- <thead style="overflow: auto;">
+ <thead>
<tr>
<#list rows as row>
<th>${row}</th>
</#list>
</tr>
</thead>
- <tbody style="overflow: auto;">
+ <tbody>
<#list model as rows>
<#if (rows_index > 0)>
<tr>
@@ -49,7 +52,10 @@
</div>
<script charset="utf-8">
$(document).ready(function() {
- $('#relation').dataTable( { "scrollX" : true } );
+ $('#relation').dataTable( {
+ "scrollX" : true,
+ "dom": '<Clfrtip>'
+ } );
} );
</script>
</#macro>
[04/32] git commit: DRILL-836: Drill needs to return complex types
(e.g., map and array) as a JSON string
Posted by ja...@apache.org.
DRILL-836: Drill needs to return complex types (e.g., map and array) as a JSON string
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/fc00bc4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/fc00bc4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/fc00bc4b
Branch: refs/heads/master
Commit: fc00bc4bc20f34b24df43a30c1340e5f1f96de54
Parents: 5da52cb
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Thu Jun 12 14:50:28 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Fri Jun 13 03:49:30 2014 -0700
----------------------------------------------------------------------
.../drill/exec/physical/config/Flatten.java | 71 ++++++++++++
.../impl/project/FlattenBatchCreator.java | 42 +++++++
.../impl/project/ProjectRecordBatch.java | 26 ++++-
.../exec/planner/physical/FlattenPrel.java | 62 ++++++++++
.../physical/visitor/FlattenPrelVisitor.java | 40 +++++++
.../planner/sql/handlers/DefaultSqlHandler.java | 45 ++++++--
.../apache/drill/exec/rpc/user/UserClient.java | 6 +-
.../apache/drill/exec/rpc/user/UserServer.java | 29 +++--
.../apache/drill/exec/rpc/user/UserSession.java | 11 +-
.../java/org/apache/drill/PlanningBase.java | 4 +-
.../exec/physical/impl/TestOptiqPlans.java | 2 +-
.../drill/exec/proto/SchemaUserProtos.java | 7 ++
.../apache/drill/exec/proto/UserBitShared.java | 15 ++-
.../org/apache/drill/exec/proto/UserProtos.java | 114 ++++++++++++++++---
.../exec/proto/beans/CoreOperatorType.java | 4 +-
.../exec/proto/beans/UserToBitHandshake.java | 23 ++++
protocol/src/main/protobuf/User.proto | 49 ++++----
protocol/src/main/protobuf/UserBitShared.proto | 1 +
18 files changed, 471 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
new file mode 100644
index 0000000..2bcaab3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Flatten.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.AbstractSingle;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("flatten")
+public class Flatten extends AbstractSingle {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Flatten.class);
+
+ @JsonCreator
+ public Flatten(@JsonProperty("child") PhysicalOperator child) {
+ super(child);
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E{
+ return physicalVisitor.visitOp(this, value);
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ return new OperatorCost(0, 0, 1000, child.getSize().getRecordCount());
+ }
+
+ @Override
+ public Size getSize() {
+ return child.getSize();
+ }
+
+ @Override
+ protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
+ return new Flatten(child);
+ }
+
+ @Override
+ public SelectionVectorMode getSVMode() {
+ return child.getSVMode();
+ }
+
+ @Override
+ public int getOperatorType() {
+ return CoreOperatorType.FLATTEN_VALUE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java
new file mode 100644
index 0000000..9bea73c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/FlattenBatchCreator.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.project;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.Flatten;
+import org.apache.drill.exec.physical.config.Project;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+
+import com.google.common.base.Preconditions;
+
+public class FlattenBatchCreator implements BatchCreator<Flatten> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, Flatten flatten, List<RecordBatch> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.size() == 1);
+ return new ProjectRecordBatch(new Project(null, flatten.getChild()),
+ children.iterator().next(),
+ context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 93cd19d..61c256b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -18,17 +18,20 @@
package org.apache.drill.exec.physical.impl.project;
import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
+import org.apache.drill.common.expression.ConvertExpression;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.FunctionCallFactory;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
@@ -48,7 +51,6 @@ import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.ValueVector;
import com.carrotsearch.hppc.IntOpenHashSet;
@@ -170,7 +172,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
protected void setupNewSchema() throws SchemaChangeException{
this.allocationVectors = Lists.newArrayList();
container.clear();
- final List<NamedExpression> exprs = popConfig.getExprs();
+ final List<NamedExpression> exprs = getExpressionList();
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
@@ -250,5 +252,23 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
}
}
+ private List<NamedExpression> getExpressionList() {
+ if (popConfig.getExprs() != null) {
+ return popConfig.getExprs();
+ }
+
+ List<NamedExpression> exprs = Lists.newArrayList();
+ for (MaterializedField field : incoming.getSchema()) {
+ if (Types.isComplex(field.getType())) {
+ exprs.add(new NamedExpression(
+ FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", field.getPath(), ExpressionPosition.UNKNOWN),
+ new FieldReference(field.getPath()))
+ );
+ } else {
+ exprs.add(new NamedExpression(field.getPath(), new FieldReference(field.getPath())));
+ }
+ }
+ return exprs;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
new file mode 100644
index 0000000..f541422
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.Flatten;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.eigenbase.rel.SingleRel;
+
+public class FlattenPrel extends SingleRel implements Prel {
+
+ public FlattenPrel(Prel phyRelNode) {
+ super(phyRelNode.getCluster(), phyRelNode.getTraitSet(), phyRelNode);
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ Flatten p = new Flatten(((Prel) getChild()).getPhysicalOperator(creator));
+ p.setOperatorId(creator.getOperatorId(this));
+ return p;
+ }
+
+ @Override
+ public Iterator<Prel> iterator() {
+ return PrelUtil.iter(getChild());
+ }
+
+ @Override
+ public SelectionVectorMode[] getSupportedEncodings() {
+ return SelectionVectorMode.DEFAULT;
+ }
+
+ @Override
+ public SelectionVectorMode getEncoding() {
+ return SelectionVectorMode.NONE;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitPrel(this, value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java
new file mode 100644
index 0000000..5892782
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FlattenPrelVisitor.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical.visitor;
+
+import java.util.Collections;
+
+import org.apache.drill.exec.planner.physical.FlattenPrel;
+import org.apache.drill.exec.planner.physical.Prel;
+import org.apache.drill.exec.planner.physical.ScreenPrel;
+import org.eigenbase.rel.RelNode;
+
+public class FlattenPrelVisitor extends BasePrelVisitor<Prel, Void, RuntimeException> {
+
+ private static final FlattenPrelVisitor INSTANCE = new FlattenPrelVisitor();
+
+ public static Prel addFlattenPrel(Prel prel) {
+ return prel.accept(INSTANCE, null);
+ }
+
+ @Override
+ public Prel visitScreen(ScreenPrel prel, Void value) throws RuntimeException {
+ return prel.copy(prel.getTraitSet(), Collections.singletonList((RelNode)new FlattenPrel((Prel)prel.getChild())));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index 883b039..aca0ae6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.planner.physical.DrillDistributionTrait;
import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
+import org.apache.drill.exec.planner.physical.visitor.FlattenPrelVisitor;
import org.apache.drill.exec.planner.physical.visitor.FinalColumnReorderer;
import org.apache.drill.exec.planner.physical.visitor.JoinPrelRenameVisitor;
import org.apache.drill.exec.planner.physical.visitor.RelUniqifier;
@@ -140,22 +141,46 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
RelTraitSet traits = drel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
Prel phyRelNode = (Prel) planner.transform(DrillSqlWorker.PHYSICAL_MEM_RULES, traits, drel);
- // Join might cause naming conflicts from its left and right child.
- // In such case, we have to insert Project to rename the conflicting names.
+ /* The order of the following transformation is important */
+
+ /*
+ * 1.)
+ * Join might cause naming conflicts from its left and right child.
+ * In such case, we have to insert Project to rename the conflicting names.
+ */
phyRelNode = JoinPrelRenameVisitor.insertRenameProject(phyRelNode);
- // Since our operators work via names rather than indices, we have to make to reorder any output
- // before we return data to the user as we may have accindentally shuffled things. This adds
- // a trivial project to reorder columns prior to output.
+ /*
+ * 2.)
+ * Since our operators work via names rather than indices, we have to make to reorder any
+ * output before we return data to the user as we may have accidentally shuffled things.
+ * This adds a trivial project to reorder columns prior to output.
+ */
phyRelNode = FinalColumnReorderer.addFinalColumnOrdering(phyRelNode);
- // Make sure that the no rels are repeats. This could happen in the case of querying the same table twice as Optiq may canonicalize these.
+ /* 3.)
+ * Next, we add any required selection vector removers given the supported encodings of each
+ * operator. This will ultimately move to a new trait but we're managing here for now to avoid
+ * introducing new issues in planning before the next release
+ */
+ phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
+
+ /* 4.)
+ * if the client does not support complex types (Map, Repeated)
+ * insert a project which which would convert
+ */
+ if (!context.getSession().isSupportComplexTypes()) {
+ logger.debug("Client does not support complex types, add Flatten operator.");
+ phyRelNode = FlattenPrelVisitor.addFlattenPrel(phyRelNode);
+ }
+
+ /* 5.)
+ * Finally, Make sure that the no rels are repeats.
+ * This could happen in the case of querying the same table twice as Optiq may canonicalize these.
+ */
phyRelNode = RelUniqifier.uniqifyGraph(phyRelNode);
- // the last thing we do is add any required selection vector removers given the supported encodings of each
- // operator. This will ultimately move to a new trait but we're managing here for now to avoid introducing new
- // issues in planning before the next release
- return SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
+ return phyRelNode;
}
protected PhysicalOperator convertToPop(Prel prel) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
index 0e3cc6b..277bb0c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
@@ -56,10 +56,12 @@ public class UserClient extends BasicClientWithConnection<RpcType, UserToBitHand
throws RpcException, InterruptedException {
UserToBitHandshake.Builder hsBuilder = UserToBitHandshake.newBuilder()
.setRpcVersion(UserRpcConfig.RPC_VERSION)
- .setSupportListening(true);
+ .setSupportListening(true)
+ .setSupportComplexTypes(true);
- if (props != null)
+ if (props != null) {
hsBuilder.setProperties(props);
+ }
this.connectAsClient(handler, hsBuilder.build(), endpoint.getAddress(), endpoint.getUserPort());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
index 2a4c2cc..e96ba6c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
@@ -27,12 +27,10 @@ import java.io.IOException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.impl.materialize.QueryWritableBatch;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
-import org.apache.drill.exec.proto.UserBitShared.UserCredentials;
import org.apache.drill.exec.proto.UserProtos.BitToUserHandshake;
import org.apache.drill.exec.proto.UserProtos.RequestResults;
import org.apache.drill.exec.proto.UserProtos.RpcType;
import org.apache.drill.exec.proto.UserProtos.RunQuery;
-import org.apache.drill.exec.proto.UserProtos.UserProperties;
import org.apache.drill.exec.proto.UserProtos.UserToBitHandshake;
import org.apache.drill.exec.rpc.Acks;
import org.apache.drill.exec.rpc.BasicServer;
@@ -61,7 +59,7 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
@Override
protected MessageLite getResponseDefaultInstance(int rpcType) throws RpcException {
- // a user server only expects acknowledgements on messages it creates.
+ // a user server only expects acknowledgments on messages it creates.
switch (rpcType) {
case RpcType.ACK_VALUE:
return Ack.getDefaultInstance();
@@ -76,7 +74,7 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
switch (rpcType) {
case RpcType.RUN_QUERY_VALUE:
- // logger.debug("Received query to run. Returning query handle.");
+ logger.trace("Received query to run. Returning query handle.");
try {
RunQuery query = RunQuery.PARSER.parseFrom(new ByteBufInputStream(pBody));
return new Response(RpcType.QUERY_HANDLE, worker.submitWork(connection, query));
@@ -85,7 +83,7 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
}
case RpcType.REQUEST_RESULTS_VALUE:
- // logger.debug("Received results requests. Returning empty query result.");
+ logger.trace("Received results requests. Returning empty query result.");
try {
RequestResults req = RequestResults.PARSER.parseFrom(new ByteBufInputStream(pBody));
return new Response(RpcType.QUERY_RESULT, worker.getResult(connection, req));
@@ -112,8 +110,9 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
super(channel);
}
- void setUser(UserCredentials credentials, UserProperties props) throws IOException{
- session = new UserSession(worker.getSystemOptions(), credentials, props);
+ void setUser(UserToBitHandshake inbound) throws IOException {
+ session = new UserSession(worker.getSystemOptions(), inbound.getCredentials(), inbound.getProperties());
+ session.setSupportComplexTypes(inbound.getSupportComplexTypes());
}
public UserSession getSession(){
@@ -121,7 +120,7 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
}
public void sendResult(RpcOutcomeListener<Ack> listener, QueryWritableBatch result){
-// logger.debug("Sending result to client with {}", result);
+ logger.trace("Sending result to client with {}", result);
send(listener, this, RpcType.QUERY_RESULT, result.getHeader(), Ack.class, result.getBuffers());
}
@@ -130,7 +129,6 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
return alloc;
}
-
}
@Override
@@ -140,16 +138,23 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
@Override
protected ServerHandshakeHandler<UserToBitHandshake> getHandshakeHandler(final UserClientConnection connection) {
+
return new ServerHandshakeHandler<UserToBitHandshake>(RpcType.HANDSHAKE, UserToBitHandshake.PARSER){
@Override
public MessageLite getHandshakeResponse(UserToBitHandshake inbound) throws Exception {
-// logger.debug("Handling handshake from user to bit. {}", inbound);
- if(inbound.getRpcVersion() != UserRpcConfig.RPC_VERSION) throw new RpcException(String.format("Invalid rpc version. Expected %d, actual %d.", inbound.getRpcVersion(), UserRpcConfig.RPC_VERSION));
- connection.setUser(inbound.getCredentials(), inbound.getProperties());
+ logger.trace("Handling handshake from user to bit. {}", inbound);
+ if(inbound.getRpcVersion() != UserRpcConfig.RPC_VERSION) {
+ throw new RpcException(String.format("Invalid rpc version. Expected %d, actual %d.", inbound.getRpcVersion(), UserRpcConfig.RPC_VERSION));
+ }
+
+ connection.setUser(inbound);
+
return BitToUserHandshake.newBuilder().setRpcVersion(UserRpcConfig.RPC_VERSION).build();
}
+
};
+
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
index 6ecffaf..18e365e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.rpc.user;
import java.io.IOException;
-import java.util.Iterator;
import java.util.Map;
import net.hydromatic.optiq.SchemaPlus;
@@ -38,6 +37,7 @@ public class UserSession {
private DrillUser user;
private boolean enableExchanges = true;
+ private boolean supportComplexTypes = false;
private UserCredentials credentials;
private Map<String, String> properties;
private OptionManager options;
@@ -107,4 +107,13 @@ public class UserSession {
return schema;
}
+ public boolean isSupportComplexTypes() {
+ return supportComplexTypes;
+ }
+
+ public UserSession setSupportComplexTypes(boolean supportComplexType) {
+ this.supportComplexTypes = supportComplexType;
+ return this;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index ad114ab..a819453 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -100,7 +100,7 @@ public class PlanningBase extends ExecTest{
registry.init();
final FunctionImplementationRegistry functionRegistry = new FunctionImplementationRegistry(config);
final SchemaPlus root = Frameworks.createRootSchema(false);
- registry.getSchemaFactory().registerSchemas(new UserSession(null, null, null), root);
+ registry.getSchemaFactory().registerSchemas(new UserSession(null, null, null).setSupportComplexTypes(true), root);
@@ -113,7 +113,7 @@ public class PlanningBase extends ExecTest{
context.getFunctionRegistry();
result = functionRegistry;
context.getSession();
- result = new UserSession(null, null, null);
+ result = new UserSession(null, null, null).setSupportComplexTypes(true);
context.getCurrentEndpoint();
result = DrillbitEndpoint.getDefaultInstance();
context.getActiveEndpoints();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index 199ecfc..f6200f0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -110,7 +110,7 @@ public class TestOptiqPlans extends ExecTest {
};
RemoteServiceSet lss = RemoteServiceSet.getLocalServiceSet();
DrillbitContext bitContext = new DrillbitContext(DrillbitEndpoint.getDefaultInstance(), context, coord, controller, com, cache, workBus, new LocalPStoreProvider(DrillConfig.create()));
- QueryContext qc = new QueryContext(new UserSession(null, null, null), QueryId.getDefaultInstance(), bitContext);
+ QueryContext qc = new QueryContext(new UserSession(null, null, null).setSupportComplexTypes(true), QueryId.getDefaultInstance(), bitContext);
PhysicalPlanReader reader = bitContext.getPlanReader();
LogicalPlan plan = reader.readLogicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
PhysicalPlan pp = new BasicOptimizer(DrillConfig.create(), qc, connection).optimize(new BasicOptimizer.BasicOptimizationContext(qc), plan);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java
index c984876..3b056cf 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java
@@ -278,6 +278,8 @@ public final class SchemaUserProtos
if(message.hasProperties())
output.writeObject(5, message.getProperties(), org.apache.drill.exec.proto.SchemaUserProtos.UserProperties.WRITE, false);
+ if(message.hasSupportComplexTypes())
+ output.writeBool(6, message.getSupportComplexTypes(), false);
}
public boolean isInitialized(org.apache.drill.exec.proto.UserProtos.UserToBitHandshake message)
{
@@ -334,6 +336,9 @@ public final class SchemaUserProtos
builder.setProperties(input.mergeObject(org.apache.drill.exec.proto.UserProtos.UserProperties.newBuilder(), org.apache.drill.exec.proto.SchemaUserProtos.UserProperties.MERGE));
break;
+ case 6:
+ builder.setSupportComplexTypes(input.readBool());
+ break;
default:
input.handleUnknownField(number, this);
}
@@ -379,6 +384,7 @@ public final class SchemaUserProtos
case 3: return "rpcVersion";
case 4: return "credentials";
case 5: return "properties";
+ case 6: return "supportComplexTypes";
default: return null;
}
}
@@ -395,6 +401,7 @@ public final class SchemaUserProtos
fieldMap.put("rpcVersion", 3);
fieldMap.put("credentials", 4);
fieldMap.put("properties", 5);
+ fieldMap.put("supportComplexTypes", 6);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index 105281a..c100968 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -454,6 +454,10 @@ public final class UserBitShared {
* <code>INFO_SCHEMA_SUB_SCAN = 30;</code>
*/
INFO_SCHEMA_SUB_SCAN(30, 30),
+ /**
+ * <code>FLATTEN = 31;</code>
+ */
+ FLATTEN(31, 31),
;
/**
@@ -580,6 +584,10 @@ public final class UserBitShared {
* <code>INFO_SCHEMA_SUB_SCAN = 30;</code>
*/
public static final int INFO_SCHEMA_SUB_SCAN_VALUE = 30;
+ /**
+ * <code>FLATTEN = 31;</code>
+ */
+ public static final int FLATTEN_VALUE = 31;
public final int getNumber() { return value; }
@@ -617,6 +625,7 @@ public final class UserBitShared {
case 28: return TEXT_SUB_SCAN;
case 29: return JSON_SUB_SCAN;
case 30: return INFO_SCHEMA_SUB_SCAN;
+ case 31: return FLATTEN;
default: return null;
}
}
@@ -16439,7 +16448,7 @@ public final class UserBitShared {
"*/\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010P" +
"HYSICAL\020\003*k\n\rFragmentState\022\013\n\007SENDING\020\000\022" +
"\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n",
- "\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005*\345" +
+ "\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005*\362" +
"\004\n\020CoreOperatorType\022\021\n\rSINGLE_SENDER\020\000\022\024" +
"\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH" +
"_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOI" +
@@ -16455,8 +16464,8 @@ public final class UserBitShared {
"K_SUB_SCAN\020\030\022\022\n\016PARQUET_WRITER\020\031\022\023\n\017DIRE" +
"CT_SUB_SCAN\020\032\022\017\n\013TEXT_WRITER\020\033\022\021\n\rTEXT_S" +
"UB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN\020\035\022\030\n\024INFO_SCH" +
- "EMA_SUB_SCAN\020\036B.\n\033org.apache.drill.exec." +
- "protoB\rUserBitSharedH\001"
+ "EMA_SUB_SCAN\020\036\022\013\n\007FLATTEN\020\037B.\n\033org.apach" +
+ "e.drill.exec.protoB\rUserBitSharedH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/java/org/apache/drill/exec/proto/UserProtos.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserProtos.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserProtos.java
index d9f4c20..048bd20 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserProtos.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserProtos.java
@@ -1669,6 +1669,16 @@ public final class UserProtos {
* <code>optional .exec.user.UserProperties properties = 5;</code>
*/
org.apache.drill.exec.proto.UserProtos.UserPropertiesOrBuilder getPropertiesOrBuilder();
+
+ // optional bool support_complex_types = 6 [default = false];
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ boolean hasSupportComplexTypes();
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ boolean getSupportComplexTypes();
}
/**
* Protobuf type {@code exec.user.UserToBitHandshake}
@@ -1768,6 +1778,11 @@ public final class UserProtos {
bitField0_ |= 0x00000010;
break;
}
+ case 48: {
+ bitField0_ |= 0x00000020;
+ supportComplexTypes_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -1900,12 +1915,29 @@ public final class UserProtos {
return properties_;
}
+ // optional bool support_complex_types = 6 [default = false];
+ public static final int SUPPORT_COMPLEX_TYPES_FIELD_NUMBER = 6;
+ private boolean supportComplexTypes_;
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ public boolean hasSupportComplexTypes() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ public boolean getSupportComplexTypes() {
+ return supportComplexTypes_;
+ }
+
private void initFields() {
channel_ = org.apache.drill.exec.proto.UserBitShared.RpcChannel.USER;
supportListening_ = false;
rpcVersion_ = 0;
credentials_ = org.apache.drill.exec.proto.UserBitShared.UserCredentials.getDefaultInstance();
properties_ = org.apache.drill.exec.proto.UserProtos.UserProperties.getDefaultInstance();
+ supportComplexTypes_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -1940,6 +1972,9 @@ public final class UserProtos {
if (((bitField0_ & 0x00000010) == 0x00000010)) {
output.writeMessage(5, properties_);
}
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeBool(6, supportComplexTypes_);
+ }
getUnknownFields().writeTo(output);
}
@@ -1969,6 +2004,10 @@ public final class UserProtos {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(5, properties_);
}
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(6, supportComplexTypes_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -2105,6 +2144,8 @@ public final class UserProtos {
propertiesBuilder_.clear();
}
bitField0_ = (bitField0_ & ~0x00000010);
+ supportComplexTypes_ = false;
+ bitField0_ = (bitField0_ & ~0x00000020);
return this;
}
@@ -2161,6 +2202,10 @@ public final class UserProtos {
} else {
result.properties_ = propertiesBuilder_.build();
}
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.supportComplexTypes_ = supportComplexTypes_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -2192,6 +2237,9 @@ public final class UserProtos {
if (other.hasProperties()) {
mergeProperties(other.getProperties());
}
+ if (other.hasSupportComplexTypes()) {
+ setSupportComplexTypes(other.getSupportComplexTypes());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -2561,6 +2609,39 @@ public final class UserProtos {
return propertiesBuilder_;
}
+ // optional bool support_complex_types = 6 [default = false];
+ private boolean supportComplexTypes_ ;
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ public boolean hasSupportComplexTypes() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ public boolean getSupportComplexTypes() {
+ return supportComplexTypes_;
+ }
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ public Builder setSupportComplexTypes(boolean value) {
+ bitField0_ |= 0x00000020;
+ supportComplexTypes_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool support_complex_types = 6 [default = false];</code>
+ */
+ public Builder clearSupportComplexTypes() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ supportComplexTypes_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:exec.user.UserToBitHandshake)
}
@@ -4250,25 +4331,26 @@ public final class UserProtos {
"\032\023UserBitShared.proto\"&\n\010Property\022\013\n\003key" +
"\030\001 \002(\t\022\r\n\005value\030\002 \002(\t\"9\n\016UserProperties\022" +
"\'\n\nproperties\030\001 \003(\0132\023.exec.user.Property" +
- "\"\326\001\n\022UserToBitHandshake\022.\n\007channel\030\001 \001(\016" +
+ "\"\374\001\n\022UserToBitHandshake\022.\n\007channel\030\001 \001(\016" +
"2\027.exec.shared.RpcChannel:\004USER\022\031\n\021suppo" +
"rt_listening\030\002 \001(\010\022\023\n\013rpc_version\030\003 \001(\005\022" +
"1\n\013credentials\030\004 \001(\0132\034.exec.shared.UserC" +
"redentials\022-\n\nproperties\030\005 \001(\0132\031.exec.us" +
- "er.UserProperties\"S\n\016RequestResults\022&\n\010q",
- "uery_id\030\001 \001(\0132\024.exec.shared.QueryId\022\031\n\021m" +
- "aximum_responses\030\002 \001(\005\"q\n\010RunQuery\0221\n\014re" +
- "sults_mode\030\001 \001(\0162\033.exec.user.QueryResult" +
- "sMode\022$\n\004type\030\002 \001(\0162\026.exec.shared.QueryT" +
- "ype\022\014\n\004plan\030\003 \001(\t\")\n\022BitToUserHandshake\022" +
- "\023\n\013rpc_version\030\002 \001(\005*\270\001\n\007RpcType\022\r\n\tHAND" +
- "SHAKE\020\000\022\007\n\003ACK\020\001\022\013\n\007GOODBYE\020\002\022\r\n\tRUN_QUE" +
- "RY\020\003\022\020\n\014CANCEL_QUERY\020\004\022\023\n\017REQUEST_RESULT" +
- "S\020\005\022\020\n\014QUERY_RESULT\020\006\022\020\n\014QUERY_HANDLE\020\007\022" +
- "\026\n\022REQ_META_FUNCTIONS\020\010\022\026\n\022RESP_FUNCTION",
- "_LIST\020\t*#\n\020QueryResultsMode\022\017\n\013STREAM_FU" +
- "LL\020\001B+\n\033org.apache.drill.exec.protoB\nUse" +
- "rProtosH\001"
+ "er.UserProperties\022$\n\025support_complex_typ",
+ "es\030\006 \001(\010:\005false\"S\n\016RequestResults\022&\n\010que" +
+ "ry_id\030\001 \001(\0132\024.exec.shared.QueryId\022\031\n\021max" +
+ "imum_responses\030\002 \001(\005\"q\n\010RunQuery\0221\n\014resu" +
+ "lts_mode\030\001 \001(\0162\033.exec.user.QueryResultsM" +
+ "ode\022$\n\004type\030\002 \001(\0162\026.exec.shared.QueryTyp" +
+ "e\022\014\n\004plan\030\003 \001(\t\")\n\022BitToUserHandshake\022\023\n" +
+ "\013rpc_version\030\002 \001(\005*\270\001\n\007RpcType\022\r\n\tHANDSH" +
+ "AKE\020\000\022\007\n\003ACK\020\001\022\013\n\007GOODBYE\020\002\022\r\n\tRUN_QUERY" +
+ "\020\003\022\020\n\014CANCEL_QUERY\020\004\022\023\n\017REQUEST_RESULTS\020" +
+ "\005\022\020\n\014QUERY_RESULT\020\006\022\020\n\014QUERY_HANDLE\020\007\022\026\n",
+ "\022REQ_META_FUNCTIONS\020\010\022\026\n\022RESP_FUNCTION_L" +
+ "IST\020\t*#\n\020QueryResultsMode\022\017\n\013STREAM_FULL" +
+ "\020\001B+\n\033org.apache.drill.exec.protoB\nUserP" +
+ "rotosH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4292,7 +4374,7 @@ public final class UserProtos {
internal_static_exec_user_UserToBitHandshake_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_exec_user_UserToBitHandshake_descriptor,
- new java.lang.String[] { "Channel", "SupportListening", "RpcVersion", "Credentials", "Properties", });
+ new java.lang.String[] { "Channel", "SupportListening", "RpcVersion", "Credentials", "Properties", "SupportComplexTypes", });
internal_static_exec_user_RequestResults_descriptor =
getDescriptor().getMessageTypes().get(3);
internal_static_exec_user_RequestResults_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
index 3690625..abd7b78 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
@@ -52,7 +52,8 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
TEXT_WRITER(27),
TEXT_SUB_SCAN(28),
JSON_SUB_SCAN(29),
- INFO_SCHEMA_SUB_SCAN(30);
+ INFO_SCHEMA_SUB_SCAN(30),
+ FLATTEN(31);
public final int number;
@@ -101,6 +102,7 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
case 28: return TEXT_SUB_SCAN;
case 29: return JSON_SUB_SCAN;
case 30: return INFO_SCHEMA_SUB_SCAN;
+ case 31: return FLATTEN;
default: return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/java/org/apache/drill/exec/proto/beans/UserToBitHandshake.java
----------------------------------------------------------------------
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/UserToBitHandshake.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/UserToBitHandshake.java
index 70235b1..67ac4e5 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/UserToBitHandshake.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/UserToBitHandshake.java
@@ -46,12 +46,14 @@ public final class UserToBitHandshake implements Externalizable, Message<UserToB
static final UserToBitHandshake DEFAULT_INSTANCE = new UserToBitHandshake();
+ static final Boolean DEFAULT_SUPPORT_COMPLEX_TYPES = new Boolean(false);
private RpcChannel channel;
private Boolean supportListening;
private int rpcVersion;
private UserCredentials credentials;
private UserProperties properties;
+ private Boolean supportComplexTypes = DEFAULT_SUPPORT_COMPLEX_TYPES;
public UserToBitHandshake()
{
@@ -125,6 +127,19 @@ public final class UserToBitHandshake implements Externalizable, Message<UserToB
return this;
}
+ // supportComplexTypes
+
+ public Boolean getSupportComplexTypes()
+ {
+ return supportComplexTypes;
+ }
+
+ public UserToBitHandshake setSupportComplexTypes(Boolean supportComplexTypes)
+ {
+ this.supportComplexTypes = supportComplexTypes;
+ return this;
+ }
+
// java serialization
public void readExternal(ObjectInput in) throws IOException
@@ -196,6 +211,9 @@ public final class UserToBitHandshake implements Externalizable, Message<UserToB
message.properties = input.mergeObject(message.properties, UserProperties.getSchema());
break;
+ case 6:
+ message.supportComplexTypes = input.readBool();
+ break;
default:
input.handleUnknownField(number, this);
}
@@ -221,6 +239,9 @@ public final class UserToBitHandshake implements Externalizable, Message<UserToB
if(message.properties != null)
output.writeObject(5, message.properties, UserProperties.getSchema(), false);
+
+ if(message.supportComplexTypes != null && message.supportComplexTypes != DEFAULT_SUPPORT_COMPLEX_TYPES)
+ output.writeBool(6, message.supportComplexTypes, false);
}
public String getFieldName(int number)
@@ -232,6 +253,7 @@ public final class UserToBitHandshake implements Externalizable, Message<UserToB
case 3: return "rpcVersion";
case 4: return "credentials";
case 5: return "properties";
+ case 6: return "supportComplexTypes";
default: return null;
}
}
@@ -250,6 +272,7 @@ public final class UserToBitHandshake implements Externalizable, Message<UserToB
__fieldMap.put("rpcVersion", 3);
__fieldMap.put("credentials", 4);
__fieldMap.put("properties", 5);
+ __fieldMap.put("supportComplexTypes", 6);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/protobuf/User.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/User.proto b/protocol/src/main/protobuf/User.proto
index ea12323..6c41a37 100644
--- a/protocol/src/main/protobuf/User.proto
+++ b/protocol/src/main/protobuf/User.proto
@@ -7,28 +7,23 @@ option optimize_for = SPEED;
import "SchemaDef.proto";
import "UserBitShared.proto";
-
-
////// UserToBit RPC ///////
enum RpcType {
- HANDSHAKE = 0;
- ACK = 1;
- GOODBYE = 2;
-
- // user to bit
- RUN_QUERY = 3;
- CANCEL_QUERY = 4;
- REQUEST_RESULTS = 5;
-
-
- // bit to user
- QUERY_RESULT = 6;
- QUERY_HANDLE = 7;
-
- REQ_META_FUNCTIONS = 8;
- RESP_FUNCTION_LIST = 9;
-
-
+ HANDSHAKE = 0;
+ ACK = 1;
+ GOODBYE = 2;
+
+ // user to bit
+ RUN_QUERY = 3;
+ CANCEL_QUERY = 4;
+ REQUEST_RESULTS = 5;
+
+ // bit to user
+ QUERY_RESULT = 6;
+ QUERY_HANDLE = 7;
+
+ REQ_META_FUNCTIONS = 8;
+ RESP_FUNCTION_LIST = 9;
}
message Property {
@@ -46,6 +41,7 @@ message UserToBitHandshake {
optional int32 rpc_version = 3;
optional exec.shared.UserCredentials credentials = 4;
optional UserProperties properties = 5;
+ optional bool support_complex_types = 6 [default = false];
}
message RequestResults {
@@ -60,16 +56,11 @@ message RunQuery {
}
enum QueryResultsMode {
- STREAM_FULL = 1; // Server will inform the client regularly on the status of the query. Once the query is completed, service will inform the client as each query chunk is made available.
- // STREAM_FIRST = 2; // Server will inform the client regularly on the status of the query. Once the query is completed, server will inform the client of the first query chunk.
- // QUERY_FOR_STATUS = 3; // Client will need to query for status of query.
+ STREAM_FULL = 1; // Server will inform the client regularly on the status of the query. Once the query is completed, service will inform the client as each query chunk is made available.
+ // STREAM_FIRST = 2; // Server will inform the client regularly on the status of the query. Once the query is completed, server will inform the client of the first query chunk.
+ // QUERY_FOR_STATUS = 3; // Client will need to query for status of query.
}
-
message BitToUserHandshake {
- optional int32 rpc_version = 2;
+ optional int32 rpc_version = 2;
}
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc00bc4b/protocol/src/main/protobuf/UserBitShared.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index 4bafeb8..b754ee5 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -197,4 +197,5 @@ enum CoreOperatorType {
TEXT_SUB_SCAN = 28;
JSON_SUB_SCAN = 29;
INFO_SCHEMA_SUB_SCAN = 30;
+ FLATTEN = 31;
}
[26/32] git commit: Update ControlConnection so that we don't block
on backup. Control messages should be allowed in band.
Posted by ja...@apache.org.
Update ControlConnection so that we don't block on backup. Control messages should be allowed in band.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c8db3462
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c8db3462
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c8db3462
Branch: refs/heads/master
Commit: c8db3462d7403094f2ab1578d9bc5e4bec8a55d1
Parents: 6f54ab0
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Jun 16 08:46:03 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jun 16 12:44:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/rpc/control/ControlConnection.java | 5 +++++
.../org/apache/drill/exec/rpc/control/ControlTunnel.java | 8 ++++----
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c8db3462/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlConnection.java
index af0368a..c03b7c2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlConnection.java
@@ -63,6 +63,11 @@ public class ControlConnection extends RemoteConnection {
bus.send(outcomeListener, this, rpcType, protobufBody, clazz, dataBodies);
}
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> void sendUnsafe(RpcOutcomeListener<RECEIVE> outcomeListener,
+ RpcType rpcType, SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ bus.send(outcomeListener, this, rpcType, protobufBody, clazz, true, dataBodies);
+ }
+
public void disable() {
active = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c8db3462/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlTunnel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlTunnel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlTunnel.java
index d0f4bde..432acab 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlTunnel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlTunnel.java
@@ -41,7 +41,7 @@ public class ControlTunnel {
this.manager = manager;
this.endpoint = endpoint;
}
-
+
public DrillbitEndpoint getEndpoint(){
return manager.getEndpoint();
}
@@ -50,13 +50,13 @@ public class ControlTunnel {
SendFragment b = new SendFragment(outcomeListener, fragment);
manager.runCommand(b);
}
-
+
public DrillRpcFuture<Ack> cancelFragment(FragmentHandle handle){
CancelFragment b = new CancelFragment(handle);
manager.runCommand(b);
return b.getFuture();
}
-
+
public DrillRpcFuture<Ack> sendFragmentStatus(FragmentStatus status){
SendFragmentStatus b = new SendFragmentStatus(status);
manager.runCommand(b);
@@ -80,7 +80,7 @@ public class ControlTunnel {
@Override
public void doRpcCall(RpcOutcomeListener<Ack> outcomeListener, ControlConnection connection) {
- connection.send(outcomeListener, RpcType.REQ_FRAGMENT_STATUS, status, Ack.class);
+ connection.sendUnsafe(outcomeListener, RpcType.REQ_FRAGMENT_STATUS, status, Ack.class);
}
}