You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/03/11 22:57:29 UTC
[1/5] git commit: DRILL-396: QuerySubmitter enhancements
Repository: incubator-drill
Updated Branches:
refs/heads/master 08a9a90d9 -> 2d10afd4b
DRILL-396: QuerySubmitter enhancements
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2ed7a678
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2ed7a678
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2ed7a678
Branch: refs/heads/master
Commit: 2ed7a678b1caef67279a775d7e8b217e3e18041f
Parents: 08a9a90
Author: Steven Phillips <sp...@maprtech.com>
Authored: Tue Mar 11 08:26:13 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 08:26:13 2014 -0700
----------------------------------------------------------------------
.../drill/exec/client/QuerySubmitter.java | 101 +++++++++++++++----
.../org/apache/drill/exec/util/VectorUtil.java | 34 +++++++
2 files changed, 116 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ed7a678/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index b07b3ae..0253069 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -20,10 +20,13 @@ package org.apache.drill.exec.client;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.coord.ZKClusterCoordinator;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -64,33 +67,47 @@ public class QuerySubmitter {
jc.usage();
System.exit(0);
}
- System.exit(submitter.submitQuery(o.location, o.planType, o.zk, o.local, o.bits));
+
+ System.exit(submitter.submitQuery(o.location, o.queryString, o.planType, o.zk, o.local, o.bits, o.format));
}
static class Options {
- @Parameter(names = {"-f"}, description = "file containing plan", required=true)
+ @Parameter(names = {"-f, --file"}, description = "file containing plan", required=false)
public String location = null;
- @Parameter(names = {"-t"}, description = "type of plan, logical/physical", required=true)
+ @Parameter(names = {"-q", "-e", "--query"}, description = "query string", required = false)
+ public String queryString = null;
+
+ @Parameter(names = {"-t", "--type"}, description = "type of query, sql/logical/physical", required=true)
public String planType;
- @Parameter(names = {"-zk"}, description = "zookeeper connect string.", required=false)
+ @Parameter(names = {"-z", "--zookeeper"}, description = "zookeeper connect string.", required=false)
public String zk = "localhost:2181";
- @Parameter(names = {"-local"}, description = "run query in local mode", required=false)
+ @Parameter(names = {"-l", "--local"}, description = "run query in local mode", required=false)
public boolean local;
- @Parameter(names = "-bits", description = "number of drillbits to run. local mode only", required=false)
+ @Parameter(names = {"-b", "--bits"}, description = "number of drillbits to run. local mode only", required=false)
public int bits = 1;
- @Parameter(names = {"-h", "-help", "--help"}, description = "show usage", help=true)
+ @Parameter(names = {"-h", "--help"}, description = "show usage", help=true)
public boolean help = false;
+
+ @Parameter(names = {"--format"}, description = "output format, csv,tsv,table", required = false)
+ public String format = "table";
}
- public int submitQuery(String planLocation, String type, String zkQuorum, boolean local, int bits) throws Exception {
+ public enum Format {
+ TSV, CSV, TABLE
+ }
+
+ public int submitQuery(String planLocation, String queryString, String type, String zkQuorum, boolean local, int bits, String format) throws Exception {
DrillConfig config = DrillConfig.create();
DrillClient client = null;
-
+
+ Preconditions.checkArgument(!(planLocation == null && queryString == null), "Must provide either query file or query string");
+ Preconditions.checkArgument(!(planLocation != null && queryString != null), "Must provide either query file or query string, not both");
+
try{
if (local) {
RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
@@ -106,29 +123,62 @@ public class QuerySubmitter {
client = new DrillClient(config, clusterCoordinator);
}
client.connect();
- QueryResultsListener listener = new QueryResultsListener();
- String plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
+ QueryResultsListener listener;
+ String plan;
+ if (queryString == null) {
+ plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
+ } else {
+ plan = queryString;
+ }
+ String[] queries;
UserProtos.QueryType queryType;
type = type.toLowerCase();
switch(type) {
case "sql":
queryType = UserProtos.QueryType.SQL;
+ queries = plan.split(";");
break;
case "logical":
queryType = UserProtos.QueryType.LOGICAL;
+ queries = new String[]{ plan };
break;
case "physical":
queryType = UserProtos.QueryType.PHYSICAL;
+ queries = new String[]{ plan };
break;
default:
System.out.println("Invalid query type: " + type);
return -1;
}
+ Format outputFormat;
+ format = format.toLowerCase();
+ switch(format) {
+ case "csv":
+ outputFormat = Format.CSV;
+ break;
+ case "tsv":
+ outputFormat = Format.TSV;
+ break;
+ case "table":
+ outputFormat = Format.TABLE;
+ break;
+ default:
+ System.out.println("Invalid format type: " + format);
+ return -1;
+ }
Stopwatch watch = new Stopwatch();
- watch.start();
- client.runQuery(queryType, plan, listener);
- int rows = listener.await();
- System.out.println(String.format("Got %d record%s in %f seconds", rows, rows > 1 ? "s" : "", (float)watch.elapsed(TimeUnit.MILLISECONDS) / (float)1000));
+ for (String query : queries) {
+ listener = new QueryResultsListener(outputFormat);
+ watch.start();
+ client.runQuery(queryType, query, listener);
+ int rows = listener.await();
+ System.out.println(String.format("%d record%s selected (%f seconds)", rows, rows > 1 ? "s" : "", (float) watch.elapsed(TimeUnit.MILLISECONDS) / (float) 1000));
+ if (query != queries[queries.length - 1]) {
+ System.out.println();
+ }
+ watch.stop();
+ watch.reset();
+ }
return 0;
}finally{
if(client != null) client.close();
@@ -139,7 +189,11 @@ public class QuerySubmitter {
AtomicInteger count = new AtomicInteger();
private CountDownLatch latch = new CountDownLatch(1);
RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
- int width;
+ Format format;
+
+ public QueryResultsListener(Format format) {
+ this.format = format;
+ }
@Override
public void submissionFailed(RpcException ex) {
@@ -157,12 +211,21 @@ public class QuerySubmitter {
} catch (SchemaChangeException e) {
submissionFailed(new RpcException(e));
}
-
- VectorUtil.showVectorAccessibleContent(loader);
+
+ switch(format) {
+ case TABLE:
+ VectorUtil.showVectorAccessibleContent(loader);
+ break;
+ case TSV:
+ VectorUtil.showVectorAccessibleContent(loader, "\t");
+ break;
+ case CSV:
+ VectorUtil.showVectorAccessibleContent(loader, ",");
+ break;
+ }
}
if (result.getHeader().getIsLastChunk()) {
- //System.out.println(StringUtils.repeat("-", width*17 + 1));
latch.countDown();
}
result.release();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ed7a678/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index 553e50d..a35bb5f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -29,6 +29,40 @@ import com.beust.jcommander.internal.Lists;
public class VectorUtil {
+ public static void showVectorAccessibleContent(VectorAccessible va, final String delimiter) {
+
+ int rows = va.getRecordCount();
+ List<String> columns = Lists.newArrayList();
+ for (VectorWrapper vw : va) {
+ columns.add(vw.getValueVector().getField().getName());
+ }
+
+ int width = columns.size();
+ for (String column : columns) {
+ System.out.printf("%s%s",column, column == columns.get(width - 1) ? "\n" : delimiter);
+ }
+ for (int row = 0; row < rows; row++) {
+ int columnCounter = 0;
+ for (VectorWrapper vw : va) {
+ boolean lastColumn = columnCounter == width - 1;
+ Object o = vw.getValueVector().getAccessor().getObject(row);
+ if (o == null) {
+ //null value
+ String value = "null";
+ System.out.printf("%s%s", value, lastColumn ? "\n" : delimiter);
+ }
+ else if (o instanceof byte[]) {
+ String value = new String((byte[]) o);
+ System.out.printf("%s%s", value, lastColumn ? "\n" : delimiter);
+ } else {
+ String value = o.toString();
+ System.out.printf("%s%s", value, lastColumn ? "\n" : delimiter);
+ }
+ columnCounter++;
+ }
+ }
+ }
+
public static void showVectorAccessibleContent(VectorAccessible va) {
int rows = va.getRecordCount();
[4/5] git commit: DRILL-398: Fix build/packaging issues with
storage-hive jars 1. Remove hive-exec dependencies from root pom.xml and move
hive-hbase dependency to contrib/storage-hive/pom.xml. Idea is to download
all required hive dependencies through s
Posted by ja...@apache.org.
DRILL-398: Fix build/packaging issues with storage-hive jars
1. Remove hive-exec dependencies from root pom.xml and move hive-hbase dependency to contrib/storage-hive/pom.xml. Idea is to download all required hive dependencies through storage-hive artifact.
2. In distribution/pom.xml exclude hive-exec jar from storage-hive dependency. Storage-hive jar already contains the shaded hive-exec.
3. In contrib/storage-hive/pom.xml add a target to unpack the classes in storage-hive.jar into contrib/storage-hive/target/classes directory. Unpacking is needed because when you run 'mvn test' from root directory, dependencies within the project (such as protocol or storage-hive) are resolved to target/classes directory instead of jar. This is by design in Maven. If we don't unpack, we get class not found errors for classes in storage-hive.jar while running tests only.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/48146084
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/48146084
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/48146084
Branch: refs/heads/master
Commit: 481460846a5c22d9c9899f5d93d443190c40061c
Parents: 17ea543
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Mar 7 10:22:11 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 14:30:22 2014 -0700
----------------------------------------------------------------------
contrib/storage-hive/pom.xml | 43 ++++++++++++++++++++++++++++++++++-
distribution/pom.xml | 7 +++++-
pom.xml | 48 ---------------------------------------
3 files changed, 48 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/48146084/contrib/storage-hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/pom.xml b/contrib/storage-hive/pom.xml
index 237d6d7..314e016 100644
--- a/contrib/storage-hive/pom.xml
+++ b/contrib/storage-hive/pom.xml
@@ -82,6 +82,21 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-hbase-handler</artifactId>
+ <version>0.12.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
@@ -114,7 +129,33 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>unpack</id>
+ <phase>package</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.drill.contrib</groupId>
+ <artifactId>storage-hive</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}/classes</outputDirectory>
+ <includes>**/**</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
-
</build>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/48146084/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index ffbac0f..06065f4 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -32,6 +32,12 @@
<groupId>org.apache.drill.contrib</groupId>
<artifactId>storage-hive</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
@@ -86,7 +92,6 @@
<exclude>**/control</exclude>
<exclude>**/*.json</exclude>
<exclude>**/hadoop-excludes.txt</exclude>
- <exclude>**/drill_hive_db/*</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/48146084/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9d8b542..967e1da 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,8 +118,6 @@
<exclude>**/*.tdd</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/target/**</exclude>
- <exclude>**/drill_hive_db/**</exclude>
- <exclude>**/drill_hive_wh/**</exclude>
</excludes>
</configuration>
</plugin>
@@ -360,52 +358,6 @@
<version>${dep.slf4j.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>0.12.0</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-hbase-handler</artifactId>
- <version>0.12.0</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>com.googlecode.jmockit</groupId>
[2/5] git commit: Drill 342/238: Clean up Parquet tests.
Posted by ja...@apache.org.
Drill 342/238: Clean up Parquet tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/822bd9c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/822bd9c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/822bd9c2
Branch: refs/heads/master
Commit: 822bd9c2dd02724385e4140cc0ab2960e8902d39
Parents: 2ed7a67
Author: Jason Altekruse <al...@gmail.com>
Authored: Tue Mar 11 08:27:53 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 08:27:53 2014 -0700
----------------------------------------------------------------------
.../store/parquet/ParquetRecordReaderTest.java | 23 +-
.../store/parquet/ParquetRecordReaderTest2.java | 742 -------------------
.../resources/parquet/parquet_nullable.json | 42 ++
.../parquet/parquet_nullable_varlen.json | 42 ++
.../parquet/parquet_selective_column_read.json | 27 +
.../src/test/resources/parquet_nullable.json | 41 -
.../test/resources/parquet_nullable_varlen.json | 41 -
.../src/test/resources/parquet_scan_screen.json | 40 -
.../parquet_scan_screen_read_entry_replace.json | 40 -
.../parquet_scan_union_screen_physical.json | 35 -
.../parquet_selective_column_read.json | 27 -
pom.xml | 2 +
12 files changed, 124 insertions(+), 978 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index 9e999e3..2b043aa 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import mockit.Injectable;
-
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
@@ -42,7 +41,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.BitControl;
import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -282,7 +281,7 @@ public class ParquetRecordReaderTest {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
populateFieldInfoMap(props);
- testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
}
// TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
@@ -295,7 +294,7 @@ public class ParquetRecordReaderTest {
ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
Object[] boolVals = {true, null, null};
props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngineEventBased(false, "/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
+ testParquetFullEngineEventBased(false, "/parquet/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
}
@Ignore
@@ -308,7 +307,7 @@ public class ParquetRecordReaderTest {
byte[] val3 = { 'l','o','n','g','e','r',' ','s','t','r','i','n','g'};
Object[] boolVals = { val, val2, val3};
props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngineEventBased(false, "/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
+ testParquetFullEngineEventBased(false, "/parquet/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
}
@Test
@@ -335,15 +334,15 @@ public class ParquetRecordReaderTest {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
TestFileGenerator.populatePigTPCHCustomerFields(props);
- String readEntries = "{path: \"/tmp/tpc-h/customer\"}";
- testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+ String readEntries = "\"/tmp/tpc-h/customer\"";
+ testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, true);
fields = new HashMap();
props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
TestFileGenerator.populatePigTPCHSupplierFields(props);
- readEntries = "{path: \"/tmp/tpc-h/supplier\"}";
- testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+ readEntries = "\"/tmp/tpc-h/supplier\"";
+ testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, true);
}
@@ -352,7 +351,7 @@ public class ParquetRecordReaderTest {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
populateFieldInfoMap(props);
- testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
}
@@ -378,7 +377,7 @@ public class ParquetRecordReaderTest {
props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, TestFileGenerator.longVals, TypeProtos.MinorType.BIGINT, props));
props.fields.put("bin", new FieldInfo("binary", "bin", -1, TestFileGenerator.binVals, TypeProtos.MinorType.VARBINARY, props));
props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, TestFileGenerator.bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- testParquetFullEngineEventBased(true, false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
+ testParquetFullEngineEventBased(true, false, "/parquet/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
}
public static void main(String[] args) throws Exception{
@@ -392,7 +391,7 @@ public class ParquetRecordReaderTest {
@Injectable UserServer.UserClientConnection connection) throws Exception {
DrillConfig c = DrillConfig.create();
FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
- FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
+ FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
// new NonStrictExpectations() {
// {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
deleted file mode 100644
index 2ab12cb..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
+++ /dev/null
@@ -1,742 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.parquet;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static parquet.column.Encoding.PLAIN;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.ByteArrayUtil;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import parquet.bytes.BytesInput;
-import parquet.column.ColumnDescriptor;
-import parquet.hadoop.ParquetFileWriter;
-import parquet.hadoop.metadata.CompressionCodecName;
-import parquet.schema.MessageType;
-import parquet.schema.MessageTypeParser;
-
-import com.beust.jcommander.internal.Lists;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.SettableFuture;
-
-public class ParquetRecordReaderTest2 {
- org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest2.class);
-
- private static final boolean VERBOSE_DEBUG = false;
-
- // { 00000001, 00000010, 00000100, 00001000, 00010000, ... }
- byte[] bitFields = {1, 2, 4, 8, 16, 32, 64, -128};
- byte allBitsTrue = -1;
- byte allBitsFalse = 0;
- int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1;
- static Object[] intVals = {-200, 100, Integer.MAX_VALUE };
- static Object[] longVals = { -5000l, 5000l, Long.MAX_VALUE};
- static Object[] floatVals = { 1.74f, Float.MAX_VALUE, Float.MIN_VALUE};
- static Object[] doubleVals = {100.45d, Double.MAX_VALUE, Double.MIN_VALUE,};
- static Object[] boolVals = {false, false, true};
- static byte[] varLen1 = {50, 51, 52, 53, 54, 55, 56, 57, 58, 59};
- static byte[] varLen2 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
- static byte[] varLen3 = {100, 99, 98};
- static Object[] binVals = { varLen3, varLen2, varLen3};
- static Object[] bin2Vals = { varLen3, varLen2, varLen1};
-
- private void populateFieldInfoMap(ParquetTestProperties props){
- props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("f", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
- props.fields.put("d", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
- props.fields.put("b", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
- props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- }
-
- private void populatePigTPCHCustomerFields(ParquetTestProperties props){
- // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
- props.fields.put("C_CUSTKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("C_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("C_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
- props.fields.put("C_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
- props.fields.put("C_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
- props.fields.put("C_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("C_MKTSEGMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("C_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- }
-
- private void populatePigTPCHSupplierFields(ParquetTestProperties props){
- // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
- props.fields.put("S_SUPPKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("S_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("S_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
- props.fields.put("S_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
- props.fields.put("S_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
- props.fields.put("S_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("S_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- }
-
- @Test
- public void testMultipleRowGroups() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populateFieldInfoMap(props);
- testParquetFullEngine(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
- }
-
- // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
- // generated using pig. Will need to find a good place to keep files like this.
- // For now I will upload it to the JIRA as an attachment.
- @Ignore
- @Test
- public void testNullableColumns() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
- Object[] boolVals = {true, null, null};
- props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngine(false, "/parquet_nullable.json", "/tmp/nullable.parquet", 1, props);
- }
-
- @Ignore
- @Test
- public void testNullableColumnsVarLen() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
- byte[] val = {'b'};
-// Object[] boolVals = { val, null, null};
-// Object[] boolVals = { null, null, null};
- Object[] boolVals = { val, val, val};
- props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngine(false, "/parquet_nullable_varlen.json", "/tmp/nullable.parquet", 1, props);
- }
-
- /**
- * Tests the attribute in a scan node to limit the columns read by a scan.
- *
- * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute.
- * @throws Exception
- */
- @Test
- public void testSelectColumnRead() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- // generate metatdata for a series of test columns, these columns are all generated in the test file
- populateFieldInfoMap(props);
- generateParquetFile("/tmp/test.parquet", props);
- fields.clear();
- // create a new object to describe the dataset expected out of the scan operation
- // the fields added below match those requested in the plan specified in parquet_selective_column_read.json
- // that is used below in the test query
- props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- testParquetFullEngineEventBased(false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
- }
-
-
- @Test
- public void testMultipleRowGroupsAndReads() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populateFieldInfoMap(props);
- String readEntries = "";
- // number of times to read the file
- int i = 3;
- for (int j = 0; j < i; j++){
- readEntries += "{path: \"/tmp/test.parquet\"}";
- if (j < i - 1)
- readEntries += ",";
- }
- testParquetFullEngineEventBased(true, "/parquet_scan_screen_read_entry_replace.json", readEntries,
- "/tmp/test.parquet", i, props, true);
- }
-
- // requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in
- @Ignore
- @Test
- public void testMultipleRowGroupsAndReadsPigError() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populatePigTPCHCustomerFields(props);
-// populatePigTPCHSupplierFields(props);
- String readEntries = "";
- // number of times to read the file
- int i = 1;
- for (int j = 0; j < i; j++){
- readEntries += "{path: \"/tmp/tpc-h/customer\"}";
- if (j < i - 1)
- readEntries += ",";
- }
- testParquetFullEngineEventBased(false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
- "/tmp/test.parquet", i, props, true);
- }
-/*
- @Test
- public void testMultipleRowGroupsEvent() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(10, 30000, DEFAULT_BYTES_PER_PAGE, fields);
- props.fields.put("a", new FieldInfo("a", "asdf", 1, new Object[3], TypeProtos.MinorType.BIGINT, props));
- testParquetFullEngineEventBased(false, "/parquet_scan_screen.json", "/tmp/out", 1, props);
- }
-
-*/
- private class ParquetTestProperties{
- int numberRowGroups;
- int recordsPerRowGroup;
- int bytesPerPage = 1024 * 1024 * 1;
- HashMap<String, FieldInfo> fields = new HashMap<>();
-
- public ParquetTestProperties(int numberRowGroups, int recordsPerRowGroup, int bytesPerPage,
- HashMap<String, FieldInfo> fields){
- this.numberRowGroups = numberRowGroups;
- this.recordsPerRowGroup = recordsPerRowGroup;
- this.bytesPerPage = bytesPerPage;
- this.fields = fields;
- }
-
- }
-
- private static class FieldInfo {
-
- String parquetType;
- String name;
- int bitLength;
- int numberOfPages;
- Object[] values;
- TypeProtos.MinorType type;
-
- FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type, ParquetTestProperties props){
- this.parquetType = parquetType;
- this.name = name;
- this.bitLength = bitLength;
- this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) props.recordsPerRowGroup) * bitLength / 8.0 / props.bytesPerPage));
- this.values = values;
- // generator is designed to use 3 values
- assert values.length == 3;
- this.type = type;
- }
- }
-
- private String getResource(String resourceName) {
- return "resource:" + resourceName;
- }
-
- public void generateParquetFile(String filename, ParquetTestProperties props) throws Exception {
-
- int currentBooleanByte = 0;
- WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
-
- Configuration configuration = new Configuration();
- configuration.set("fs.default.name", "file:///");
- //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
-
- FileSystem fs = FileSystem.get(configuration);
- Path path = new Path(filename);
- if (fs.exists(path)) fs.delete(path, false);
-
-
- String messageSchema = "message m {";
- for (FieldInfo fieldInfo : props.fields.values()) {
- messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";";
- }
- // remove the last semicolon, java really needs a join method for strings...
- // TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug
- //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
- messageSchema += "}";
-
- MessageType schema = MessageTypeParser.parseMessageType(messageSchema);
-
- CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
- ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
- w.start();
- HashMap<String, Integer> columnValuesWritten = new HashMap();
- int valsWritten;
- for (int k = 0; k < props.numberRowGroups; k++){
- w.startBlock(1);
- currentBooleanByte = 0;
- booleanBitCounter.reset();
-
- for (FieldInfo fieldInfo : props.fields.values()) {
-
- if ( ! columnValuesWritten.containsKey(fieldInfo.name)){
- columnValuesWritten.put((String) fieldInfo.name, 0);
- valsWritten = 0;
- } else {
- valsWritten = columnValuesWritten.get(fieldInfo.name);
- }
-
- String[] path1 = {(String) fieldInfo.name};
- ColumnDescriptor c1 = schema.getColumnDescription(path1);
-
- w.startColumn(c1, props.recordsPerRowGroup, codec);
- int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages);
- byte[] bytes;
- // for variable length binary fields
- int bytesNeededToEncodeLength = 4;
- if ((int) fieldInfo.bitLength > 0) {
- bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)];
- } else {
- // the twelve at the end is to account for storing a 4 byte length with each value
- int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
- // used for the case where there is a number of values in this row group that is not divisible by 3
- int leftOverBytes = 0;
- if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + bytesNeededToEncodeLength;
- if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + bytesNeededToEncodeLength;
- bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes];
- }
- int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0));
- int bytesWritten = 0;
- for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) {
- for (int i = 0; i < valsPerPage; i++) {
- //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
- if (fieldInfo.values[0] instanceof Boolean) {
-
- bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val] & ((boolean) fieldInfo.values[valsWritten % 3]
- ? allBitsTrue : allBitsFalse);
- booleanBitCounter.increment();
- if (booleanBitCounter.val == 0) {
- currentBooleanByte++;
- }
- valsWritten++;
- if (currentBooleanByte > bytesPerPage) break;
- } else {
- if (fieldInfo.values[valsWritten % 3] instanceof byte[]){
- System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length),
- 0, bytes, bytesWritten, bytesNeededToEncodeLength);
- try{
- System.arraycopy(fieldInfo.values[valsWritten % 3],
- 0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length);
- }
- catch (Exception ex){
- Math.min(4, 5);
- }
- bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
- }
- else{
- System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]),
- 0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
- }
- valsWritten++;
- }
-
- }
- w.writeDataPage((int) (props.recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
- currentBooleanByte = 0;
- }
- w.endColumn();
- columnValuesWritten.remove((String) fieldInfo.name);
- columnValuesWritten.put((String) fieldInfo.name, valsWritten);
- }
-
- w.endBlock();
- }
- w.end(new HashMap<String, String>());
- logger.debug("Finished generating parquet file.");
- }
-
- private class ParquetResultListener implements UserResultsListener {
- private SettableFuture<Void> future = SettableFuture.create();
- int count = 0;
- RecordBatchLoader batchLoader;
-
- int batchCounter = 1;
- HashMap<String, Integer> valuesChecked = new HashMap();
- ParquetTestProperties props;
-
- ParquetResultListener(RecordBatchLoader batchLoader, ParquetTestProperties props){
- this.batchLoader = batchLoader;
- this.props = props;
- }
-
- @Override
- public void submissionFailed(RpcException ex) {
- logger.debug("Submission failed.", ex);
- future.setException(ex);
- }
-
- @Override
- public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
- logger.debug("result arrived in test batch listener.");
- if(result.getHeader().getIsLastChunk()){
- future.set(null);
- }
- int columnValCounter = 0;
- int i = 0;
- FieldInfo currentField;
- count += result.getHeader().getRowCount();
- boolean schemaChanged = false;
- try {
- schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
- } catch (SchemaChangeException e) {
- throw new RuntimeException(e);
- }
-
- int recordCount = 0;
- // print headers.
- if (schemaChanged) {
- } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector vv = vw.getValueVector();
- currentField = props.fields.get(vv.getField().getName());
- if (VERBOSE_DEBUG){
- System.out.println("\n" + (String) currentField.name);
- }
- if ( ! valuesChecked.containsKey(vv.getField().getName())){
- valuesChecked.put(vv.getField().getName(), 0);
- columnValCounter = 0;
- } else {
- columnValCounter = valuesChecked.get(vv.getField().getName());
- }
- for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
- if (VERBOSE_DEBUG){
- System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
- }
- assertField(vv, j, (TypeProtos.MinorType) currentField.type,
- currentField.values[columnValCounter % 3], (String) currentField.name + "/");
- columnValCounter++;
- }
- if (VERBOSE_DEBUG){
- System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
- }
- valuesChecked.remove(vv.getField().getName());
- valuesChecked.put(vv.getField().getName(), columnValCounter);
- }
-
- if (VERBOSE_DEBUG){
- for (i = 0; i < batchLoader.getRecordCount(); i++) {
- recordCount++;
- if (i % 50 == 0){
- System.out.println();
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getField().getName(), 20) + " ");
-
- }
- System.out.println();
- System.out.println();
- }
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
- }
- System.out.println(
-
- );
- }
- }
- batchCounter++;
- if(result.getHeader().getIsLastChunk()){
- future.set(null);
- }
- }
-
- public void getResults() throws RpcException{
- try{
- future.get();
- }catch(Throwable t){
- throw RpcException.mapException(t);
- }
- }
-
- @Override
- public void queryIdArrived(QueryId queryId) {
- }
- }
-
-
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- testParquetFullEngineEventBased(generateNew, plan, null, filename, numberOfTimesRead, props, true);
- }
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename,
- int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props,
- boolean runAsLogicalPlan) throws Exception{
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- if (generateNew) generateParquetFile(filename, props);
-
- DrillConfig config = DrillConfig.create();
-
- try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
- bit1.run();
- client.connect();
- RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
- ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props);
- long C = System.nanoTime();
- String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
- // substitute in the string for the read entries, allows reuse of the plan file for several tests
- if (readEntries != null) {
- planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
- }
- if (runAsLogicalPlan)
- client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
- else
- client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener);
- resultListener.getResults();
- for (String s : resultListener.valuesChecked.keySet()) {
- assertEquals("Record count incorrect for column: " + s,
- props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) resultListener.valuesChecked.get(s));
- logger.debug("Column {}, Values read:{}", s, resultListener.valuesChecked.get(s));
- }
- long D = System.nanoTime();
- System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
- }
- }
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngine(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- testParquetFullEngine(generateNew, plan, null, filename, numberOfTimesRead, props);
- }
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngine(boolean generateNew, String plan, String readEntries, String filename,
- int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- if (generateNew) generateParquetFile(filename, props);
-
- DrillConfig config = DrillConfig.create();
-
- try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
- long A = System.nanoTime();
- bit1.run();
- long B = System.nanoTime();
- client.connect();
- long C = System.nanoTime();
- List<QueryResultBatch> results;
- // insert a variable number of reads
- if (readEntries != null){
- results = client.runQuery(UserProtos.QueryType.LOGICAL, (Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8).replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries)));
- }
- else{
- results = client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8));
- }
-// List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
- long D = System.nanoTime();
- System.out.println(String.format("Took %f s to start drillbit", (float)(B-A) / 1E9));
- System.out.println(String.format("Took %f s to connect", (float)(C-B) / 1E9));
- System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
- //List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
- int count = 0;
-// RecordBatchLoader batchLoader = new RecordBatchLoader(new BootStrapContext(config).getAllocator());
- RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
- byte[] bytes;
-
- int batchCounter = 1;
- int columnValCounter = 0;
- int i = 0;
- FieldInfo currentField;
- HashMap<String, Integer> valuesChecked = new HashMap();
- for(QueryResultBatch b : results){
-
- count += b.getHeader().getRowCount();
- boolean schemaChanged = batchLoader.load(b.getHeader().getDef(), b.getData());
-
- int recordCount = 0;
- // print headers.
- if (schemaChanged) {
- } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector vv = vw.getValueVector();
- currentField = props.fields.get(vv.getField().getName());
- if (VERBOSE_DEBUG){
- System.out.println("\n" + (String) currentField.name);
- }
- if ( ! valuesChecked.containsKey(vv.getField().getName())){
- valuesChecked.put(vv.getField().getName(), 0);
- columnValCounter = 0;
- } else {
- columnValCounter = valuesChecked.get(vv.getField().getName());
- }
- for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
- if (VERBOSE_DEBUG){
- System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
- }
- assertField(vv, j, currentField.type,
- currentField.values[columnValCounter % 3], currentField.name + "/");
- columnValCounter++;
- }
- if (VERBOSE_DEBUG){
- System.out.println("\n" + vv.getAccessor().getValueCount());
- }
- valuesChecked.remove(vv.getField().getName());
- valuesChecked.put(vv.getField().getName(), columnValCounter);
- }
-
- if (VERBOSE_DEBUG){
- for (i = 0; i < batchLoader.getRecordCount(); i++) {
- recordCount++;
- if (i % 50 == 0){
- System.out.println();
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getField().getName(), 20) + " ");
-
- }
- System.out.println();
- System.out.println();
- }
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getAccessor().getObject(i) + "", 20) + " ");
- }
- System.out.println(
-
- );
- }
- }
- batchCounter++;
- }
- for (String s : valuesChecked.keySet()) {
- assertEquals("Record count incorrect for column: " + s, props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) valuesChecked.get(s));
- }
- assert valuesChecked.keySet().size() > 0;
- }
- }
-
- public String pad(String value, int length) {
- return pad(value, length, " ");
- }
-
- public String pad(String value, int length, String with) {
- StringBuilder result = new StringBuilder(length);
- result.append(value);
-
- while (result.length() < length) {
- result.insert(0, with);
- }
-
- return result.toString();
- }
-
- class MockOutputMutator implements OutputMutator {
- List<MaterializedField> removedFields = Lists.newArrayList();
- List<ValueVector> addFields = Lists.newArrayList();
-
- @Override
- public void removeField(MaterializedField field) throws SchemaChangeException {
- removedFields.add(field);
- }
-
- @Override
- public void addField(ValueVector vector) throws SchemaChangeException {
- addFields.add(vector);
- }
-
- @Override
- public void removeAllFields() {
- addFields.clear();
- }
-
- @Override
- public void setNewSchema() throws SchemaChangeException {
- }
-
- List<MaterializedField> getRemovedFields() {
- return removedFields;
- }
-
- List<ValueVector> getAddFields() {
- return addFields;
- }
- }
-
- private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
- assertField(valueVector, index, expectedMinorType, value, name, 0);
- }
-
- private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
-// UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
-// SchemaDefProtos.FieldDef def = metadata.getDef();
-// assertEquals(expectedMinorType, def.getMajorType().getMinorType());
-// assertEquals(name, def.getNameList().get(0).getName());
-// assertEquals(parentFieldId, def.getParentId());
-
- if (expectedMinorType == TypeProtos.MinorType.MAP) {
- return;
- }
-
- T val = (T) valueVector.getAccessor().getObject(index);
- if (val instanceof String){
- assertEquals(value, val);
- }
- else if (val instanceof byte[]) {
- assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
- } else {
- assertEquals(value, val);
- }
- }
-
- private class WrapAroundCounter {
-
- int maxVal;
- int val;
-
- public WrapAroundCounter(int maxVal) {
- this.maxVal = maxVal;
- }
-
- public int increment() {
- val++;
- if (val > maxVal) {
- val = 0;
- }
- return val;
- }
-
- public void reset() {
- val = 0;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_nullable.json b/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
new file mode 100644
index 0000000..3e09f83
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
@@ -0,0 +1,42 @@
+{
+ head:{
+ type:"APACHE_DRILL_LOGICAL",
+ version:"1",
+ generator:{
+ type:"manual",
+ info:"na"
+ }
+ },
+ storage:{
+ "dfs" :
+ {
+ "type":"named",
+ "name":"dfs"
+ }
+ },
+ query:[
+ {
+ @id:"1",
+ op:"scan",
+ memo:"initial_scan",
+ storageengine:"dfs",
+ selection: {
+ format: {type: "parquet"},
+ files: [
+ "/tmp/nullable_test.parquet"
+ ]
+ }
+ },
+ {
+ @id:"2",
+ input: 1,
+ op: "store",
+ memo: "output sink",
+ target: {
+ file: "console:///stdout"
+ }
+
+ }
+
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json b/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
new file mode 100644
index 0000000..9547f00
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
@@ -0,0 +1,42 @@
+{
+ head:{
+ type:"APACHE_DRILL_LOGICAL",
+ version:"1",
+ generator:{
+ type:"manual",
+ info:"na"
+ }
+ },
+ storage:{
+ "dfs" :
+ {
+ "type":"named",
+ "name":"dfs"
+ }
+ },
+ query:[
+ {
+ @id:"1",
+ op:"scan",
+ memo:"initial_scan",
+ storageengine:"dfs",
+ selection: {
+ format: {type: "parquet"},
+ files: [
+ "/tmp/nullable_varlen.parquet"
+ ]
+ }
+ },
+ {
+ @id:"2",
+ input: 1,
+ op: "store",
+ memo: "output sink",
+ target: {
+ file: "console:///stdout"
+ }
+
+ }
+
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json
new file mode 100644
index 0000000..515affc
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json
@@ -0,0 +1,27 @@
+{
+ head : {
+ version : 1,
+ generator : {
+ type : "manual",
+ info : "na"
+ },
+ type : "APACHE_DRILL_PHYSICAL"
+ },
+ graph : [ {
+ pop : "parquet-scan",
+ @id : 1,
+ entries : [ {
+ path : "/tmp/test.parquet"
+ } ],
+ storage : {
+ type : "file",
+ connection : "file:///"
+ },
+ columns: [ "integer", "bigInt", "bin", "bin2"]
+ }, {
+ pop : "screen",
+ @id : 2,
+ child : 1
+ } ]
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_nullable.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_nullable.json b/exec/java-exec/src/test/resources/parquet_nullable.json
deleted file mode 100644
index 5822bdb..0000000
--- a/exec/java-exec/src/test/resources/parquet_nullable.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:{
- "parquet" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- }
- },
- query:[
- {
- @id:"1",
- op:"scan",
- memo:"initial_scan",
- storageengine:"parquet",
- selection: [
- {
- path: "/tmp/nullable_test.parquet"
- }
- ]
- },
- {
- @id:"2",
- input: 1,
- op: "store",
- memo: "output sink",
- target: {
- file: "console:///stdout"
- }
-
- }
-
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_nullable_varlen.json b/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
deleted file mode 100644
index c21beec..0000000
--- a/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:{
- "parquet" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- }
- },
- query:[
- {
- @id:"1",
- op:"scan",
- memo:"initial_scan",
- storageengine:"parquet",
- selection: [
- {
- path: "/tmp/nullable_varlen.parquet"
- }
- ]
- },
- {
- @id:"2",
- input: 1,
- op: "store",
- memo: "output sink",
- target: {
- file: "console:///stdout"
- }
-
- }
-
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_scan_screen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet_scan_screen.json
deleted file mode 100644
index 3b3f86a..0000000
--- a/exec/java-exec/src/test/resources/parquet_scan_screen.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:{
- "dfs" :
- {
- "type":"file",
- "connection" : "file:///"
- }
- },
- query:[
- {
- @id:"1",
- op:"scan",
- memo:"initial_scan",
- storageengine:"dfs",
- selection: {
- format: {type: "parquet"},
- files: ["/tmp/test.parquet"]
- }
- },
- {
- @id:"2",
- input: 1,
- op: "store",
- memo: "output sink",
- target: {
- file: "console:///stdout"
- }
-
- }
-
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json b/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
deleted file mode 100644
index ab11a5f..0000000
--- a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:{
- "fs" :
- {
- "type":"file",
- "connection" : "file:///"
- }
- },
- query:[
- {
- @id:"1",
- op:"scan",
- memo:"initial_scan",
- storageengine:"fs",
- selection: {
- format: {type: "parquet"}
- files: [ &REPLACED_IN_PARQUET_TEST& ]
- }
- },
- {
- @id:"2",
- input: 1,
- op: "store",
- memo: "output sink",
- target: {
- file: "console:///stdout"
- }
-
- }
-
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json b/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
deleted file mode 100644
index 954082c..0000000
--- a/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- head : {
- type : "APACHE_DRILL_PHYSICAL",
- version : 1,
- generator : {
- type : "manual"
- }
- },
- graph : [ {
- pop : "parquet-scan",
- @id : 1,
- entries : [
- {
- path : "/tmp/testParquetFile_many_types_3"
- },
- {
- path : "/tmp/testParquetFile_many_types_3"
- }
- ],
- storageengine:{
- "type":"parquet",
- "dfsName" : "maprfs:///"
- }
- },
- {
- "@id": 2,
- "child": 1,
- "pop": "union-exchange"
- },
- {
- pop : "screen",
- @id : 3,
- child : 2
- } ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_selective_column_read.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet_selective_column_read.json
deleted file mode 100644
index 515affc..0000000
--- a/exec/java-exec/src/test/resources/parquet_selective_column_read.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- head : {
- version : 1,
- generator : {
- type : "manual",
- info : "na"
- },
- type : "APACHE_DRILL_PHYSICAL"
- },
- graph : [ {
- pop : "parquet-scan",
- @id : 1,
- entries : [ {
- path : "/tmp/test.parquet"
- } ],
- storage : {
- type : "file",
- connection : "file:///"
- },
- columns: [ "integer", "bigInt", "bin", "bin2"]
- }, {
- pop : "screen",
- @id : 2,
- child : 1
- } ]
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1bb58c0..9d8b542 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,8 @@
<exclude>**/*.proto</exclude>
<exclude>**/*.fmpp</exclude>
<exclude>**/*.tdd</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>**/target/**</exclude>
<exclude>**/drill_hive_db/**</exclude>
<exclude>**/drill_hive_wh/**</exclude>
</excludes>
[5/5] git commit: Fix group by,
removing aggregate and segment remnants. Add new simple test queries
to make sure parsing isn't broken in future. Fix binary tree so that more
than 2 binary arguments are correctly translated from RexNode. Update binary
f
Posted by ja...@apache.org.
Fix group by, removing aggregate and segment remnants. Add new simple test queries to make sure parsing isn't broken in future.
Fix binary tree so that more than 2 binary arguments are correctly translated from RexNode.
Update binary function lookup to use lowercase.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2d10afd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2d10afd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2d10afd4
Branch: refs/heads/master
Commit: 2d10afd4b1499b84084123eda1e85e59602ac475
Parents: 4814608
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Mar 11 14:33:57 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 14:57:10 2014 -0700
----------------------------------------------------------------------
.../logical/data/CollapsingAggregate.java | 73 -------------------
.../drill/common/logical/data/Segment.java | 62 ----------------
.../data/visitors/AbstractLogicalVisitor.java | 10 ---
.../logical/data/visitors/LogicalVisitor.java | 4 -
exec/java-exec/pom.xml | 6 ++
.../drill/exec/client/QuerySubmitter.java | 4 +-
.../apache/drill/exec/opt/BasicOptimizer.java | 54 +++++---------
.../drill/exec/planner/logical/DrillOptiq.java | 15 +++-
.../org/apache/drill/TestExampleQueries.java | 51 +++++++++++++
.../src/test/resources/storage-engines.json | 4 +
pom.xml | 2 +-
.../org/apache/drill/jdbc/test/JdbcTest.java | 77 ++++++++++----------
12 files changed, 132 insertions(+), 230 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java b/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java
deleted file mode 100644
index f4258fd..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import com.google.common.collect.Iterators;
-import org.apache.drill.common.expression.FieldReference;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
-import java.util.List;
-
-@JsonTypeName("collapsingaggregate")
-public class CollapsingAggregate extends SingleInputOperator{
-
- private final FieldReference within;
- private final FieldReference target;
- private final FieldReference[] carryovers;
- private final NamedExpression[] aggregations;
-
- @JsonCreator
- public CollapsingAggregate(@JsonProperty("within") FieldReference within, @JsonProperty("target") FieldReference target, @JsonProperty("carryovers") FieldReference[] carryovers, @JsonProperty("aggregations") NamedExpression[] aggregations) {
- super();
- this.within = within;
- this.target = target;
- this.carryovers = carryovers;
- this.aggregations = aggregations;
- }
-
- public FieldReference getWithin() {
- return within;
- }
-
- public FieldReference getTarget() {
- return target;
- }
-
- public FieldReference[] getCarryovers() {
- return carryovers;
- }
-
- public NamedExpression[] getAggregations() {
- return aggregations;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitCollapsingAggregate(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/Segment.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Segment.java b/common/src/main/java/org/apache/drill/common/logical/data/Segment.java
deleted file mode 100644
index 3898944..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Segment.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import com.google.common.collect.Iterators;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
-
-@JsonTypeName("segment")
-public class Segment extends SingleInputOperator{
- private final LogicalExpression[] exprs;
- private final FieldReference name;
-
- @JsonCreator
- public Segment(@JsonProperty("exprs") LogicalExpression[] exprs, @JsonProperty("ref") FieldReference name) {
- super();
- this.exprs = exprs;
- this.name = name;
- }
-
- public LogicalExpression[] getExprs() {
- return exprs;
- }
-
- public FieldReference getName() {
- return name;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitSegment(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
index bb37a14..79fc34e 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
@@ -74,11 +74,6 @@ public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implemen
}
@Override
- public T visitSegment(Segment segment, X value) throws E {
- return visitOp(segment, value);
- }
-
- @Override
public T visitGroupingAggregate(GroupingAggregate groupBy, X value) throws E {
return visitOp(groupBy, value);
}
@@ -99,11 +94,6 @@ public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implemen
}
@Override
- public T visitCollapsingAggregate(CollapsingAggregate collapsingAggregate, X value) throws E {
- return visitOp(collapsingAggregate, value);
- }
-
- @Override
public T visitWindowFrame(WindowFrame windowFrame, X value) throws E {
return visitOp(windowFrame, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
index f409453..d523541 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
@@ -18,7 +18,6 @@
package org.apache.drill.common.logical.data.visitors;
-import org.apache.drill.common.logical.data.CollapsingAggregate;
import org.apache.drill.common.logical.data.Constant;
import org.apache.drill.common.logical.data.Filter;
import org.apache.drill.common.logical.data.Flatten;
@@ -29,7 +28,6 @@ import org.apache.drill.common.logical.data.Order;
import org.apache.drill.common.logical.data.Project;
import org.apache.drill.common.logical.data.RunningAggregate;
import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Segment;
import org.apache.drill.common.logical.data.Sequence;
import org.apache.drill.common.logical.data.Store;
import org.apache.drill.common.logical.data.Transform;
@@ -48,7 +46,6 @@ public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitScan(Scan scan, EXTRA value) throws EXCEP;
public RETURN visitStore(Store store, EXTRA value) throws EXCEP;
- public RETURN visitCollapsingAggregate(CollapsingAggregate collapsingAggregate, EXTRA value) throws EXCEP;
public RETURN visitGroupingAggregate(GroupingAggregate groupBy, EXTRA value) throws EXCEP;
public RETURN visitFilter(Filter filter, EXTRA value) throws EXCEP;
public RETURN visitFlatten(Flatten flatten, EXTRA value) throws EXCEP;
@@ -59,7 +56,6 @@ public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitJoin(Join join, EXTRA value) throws EXCEP;
public RETURN visitLimit(Limit limit, EXTRA value) throws EXCEP;
public RETURN visitRunningAggregate(RunningAggregate runningAggregate, EXTRA value) throws EXCEP;
- public RETURN visitSegment(Segment segment, EXTRA value) throws EXCEP;
public RETURN visitSequence(Sequence sequence, EXTRA value) throws EXCEP;
public RETURN visitTransform(Transform transform, EXTRA value) throws EXCEP;
public RETURN visitUnion(Union union, EXTRA value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index aba336e..bb044ad 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -27,6 +27,12 @@
<version>4.1</version>
</dependency>
<dependency>
+ <groupId>pentaho</groupId>
+ <artifactId>mondrian-data-foodmart-json</artifactId>
+ <version>0.3.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
<version>4.1</version>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index 0253069..6449d93 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -190,6 +190,7 @@ public class QuerySubmitter {
private CountDownLatch latch = new CountDownLatch(1);
RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
Format format;
+ volatile Exception exception;
public QueryResultsListener(Format format) {
this.format = format;
@@ -197,7 +198,7 @@ public class QuerySubmitter {
@Override
public void submissionFailed(RpcException ex) {
- System.out.println(String.format("Query failed: %s", ex));
+ exception = ex;
latch.countDown();
}
@@ -233,6 +234,7 @@ public class QuerySubmitter {
public int await() throws Exception {
latch.await();
+ if(exception != null) throw exception;
return count.get();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index aed4802..4ea87e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -25,14 +25,11 @@ import java.util.List;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.common.logical.data.CollapsingAggregate;
import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.GroupingAggregate;
import org.apache.drill.common.logical.data.Join;
import org.apache.drill.common.logical.data.JoinCondition;
import org.apache.drill.common.logical.data.NamedExpression;
@@ -40,7 +37,6 @@ import org.apache.drill.common.logical.data.Order;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.common.logical.data.Project;
import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Segment;
import org.apache.drill.common.logical.data.SinkOperator;
import org.apache.drill.common.logical.data.Store;
import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
@@ -121,11 +117,23 @@ public class BasicOptimizer extends Optimizer{
this.logicalPlan = logicalPlan;
}
-
-
@Override
- public PhysicalOperator visitSegment(Segment segment, Object value) throws OptimizerException {
- throw new OptimizerException("Segment operators aren't currently supported besides next to a collapsing aggregate operator.");
+ public PhysicalOperator visitGroupingAggregate(GroupingAggregate groupBy, Object value) throws OptimizerException {
+
+ List<Ordering> orderDefs = Lists.newArrayList();
+
+
+ PhysicalOperator input = groupBy.getInput().accept(this, value);
+
+ if(groupBy.getKeys().length > 0){
+ for(NamedExpression e : groupBy.getKeys()){
+ orderDefs.add(new Ordering(Direction.Ascending, e.getExpr(), NullDirection.FIRST));
+ }
+ input = new Sort(input, orderDefs, false);
+ }
+
+ StreamingAggregate sa = new StreamingAggregate(input, groupBy.getKeys(), groupBy.getExprs(), 1.0f);
+ return sa;
}
@@ -146,34 +154,6 @@ public class BasicOptimizer extends Optimizer{
return new SelectionVectorRemover(new Limit(input, limit.getFirst(), limit.getLast()));
}
- @Override
- public PhysicalOperator visitCollapsingAggregate(CollapsingAggregate agg, Object value)
- throws OptimizerException {
-
- if( !(agg.getInput() instanceof Segment) ){
- throw new OptimizerException(String.format("Currently, Drill only supports CollapsingAggregate immediately preceded by a Segment. The input of this operator is %s.", agg.getInput()));
- }
- Segment segment = (Segment) agg.getInput();
-
- if(!agg.getWithin().equals(segment.getName())){
- throw new OptimizerException(String.format("Currently, Drill only supports CollapsingAggregate immediately preceded by a Segment where the CollapsingAggregate works on the defined segments. In this case, the segment has been defined based on the name %s but the collapsing aggregate is working within the field %s.", segment.getName(), agg.getWithin()));
- }
-
- // a collapsing aggregate is a currently implemented as a sort followed by a streaming aggregate.
- List<Ordering> orderDefs = Lists.newArrayList();
-
- List<NamedExpression> keys = Lists.newArrayList();
- for(LogicalExpression e : segment.getExprs()){
- if( !(e instanceof SchemaPath)) throw new OptimizerException("The basic optimizer doesn't currently support collapsing aggregate where the segment value is something other than a SchemaPath.");
- keys.add(new NamedExpression(e, new FieldReference((SchemaPath) e)));
- orderDefs.add(new Ordering(Direction.Ascending, e, NullDirection.FIRST));
- }
- Sort sort = new Sort(segment.getInput().accept(this, value), orderDefs, false);
-
- StreamingAggregate sa = new StreamingAggregate(sort, keys.toArray(new NamedExpression[keys.size()]), agg.getAggregations(), 1.0f);
- return sa;
- }
-
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 2e29bd4..13f5494 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -84,9 +84,18 @@ public class DrillOptiq {
switch (syntax) {
case BINARY:
logger.debug("Binary");
- LogicalExpression op1 = call.getOperands().get(0).accept(this);
- LogicalExpression op2 = call.getOperands().get(1).accept(this);
- return context.getRegistry().createExpression(call.getOperator().getName(), Lists.newArrayList(op1, op2));
+ final String funcName = call.getOperator().getName().toLowerCase();
+ List<LogicalExpression> args = Lists.newArrayList();
+ for(RexNode r : call.getOperands()){
+ args.add(r.accept(this));
+ }
+ args = Lists.reverse(args);
+ LogicalExpression lastArg = args.get(0);
+ for(int i = 1; i < args.size(); i++){
+ lastArg = context.getRegistry().createExpression(funcName, Lists.newArrayList(args.get(i), lastArg));
+ }
+
+ return lastArg;
case FUNCTION:
logger.debug("Function");
List<LogicalExpression> exprs = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
new file mode 100644
index 0000000..c7fde56
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.apache.drill.common.util.TestTools;
+import org.apache.drill.exec.client.QuerySubmitter;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+public class TestExampleQueries {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
+
+ @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(10000);
+
+ @Test
+ public void testSelectWithLimit() throws Exception{
+ test("select * from cp.`employee.json` limit 5");
+ }
+
+ @Test
+ public void testWhere() throws Exception{
+ test("select * from cp.`employee.json` where employee_id > 10 and employee_id < 20");
+ }
+
+ @Test
+ public void testGroupBy() throws Exception{
+ test("select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status");
+ }
+
+ private void test(String sql) throws Exception{
+ QuerySubmitter s = new QuerySubmitter();
+ s.submitQuery(null, sql, "sql", null, true, 1, "tsv");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/storage-engines.json b/exec/java-exec/src/test/resources/storage-engines.json
index 58f091e..6e4d23e 100644
--- a/exec/java-exec/src/test/resources/storage-engines.json
+++ b/exec/java-exec/src/test/resources/storage-engines.json
@@ -3,6 +3,10 @@
dfs: {
type: "file",
connection: "file:///"
+ },
+ cp: {
+ type: "file",
+ connection: "classpath:///"
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 967e1da..fc5158d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -211,7 +211,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.15</version>
<configuration>
- <argLine>-XX:MaxDirectMemorySize=4096M </argLine>
+ <argLine>-XX:MaxDirectMemorySize=6096M </argLine>
<additionalClasspathElements>
<additionalClasspathElement>./sqlparser/src/test/resources/storage-engines.json</additionalClasspathElement>
</additionalClasspathElements>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index 8810b6b..8cd682b 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.common.logical.data.CollapsingAggregate;
import org.apache.drill.common.logical.data.Filter;
import org.apache.drill.common.logical.data.Join;
import org.apache.drill.common.logical.data.Limit;
@@ -284,44 +283,44 @@ public class JdbcTest {
// .planContains("store");
}
- @Test
- public void testDistinct() throws Exception {
- JdbcAssert.withModel(MODEL, "HR").sql("select distinct deptId from emp")
- .returnsUnordered("DEPTID=null", "DEPTID=31", "DEPTID=34", "DEPTID=33")
- .planContains(CollapsingAggregate.class);
- }
-
- @Test
- public void testCountNoGroupBy() throws Exception {
- // 5 out of 6 employees have a not-null deptId
- JdbcAssert.withModel(MODEL, "HR").sql("select count(deptId) as cd, count(*) as c from emp").returns("CD=5; C=6\n")
- .planContains(CollapsingAggregate.class);
- }
-
- @Test
- public void testDistinctCountNoGroupBy() throws Exception {
- JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp").returns("C=3\n")
- .planContains(CollapsingAggregate.class);
- }
-
- @Test
- public void testDistinctCountGroupByEmpty() throws Exception {
- JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
- .planContains(CollapsingAggregate.class);
- }
-
- @Test
- public void testCountNull() throws Exception {
- JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
- .planContains(CollapsingAggregate.class);
- }
-
- @Test
- public void testCount() throws Exception {
- JdbcAssert.withModel(MODEL, "HR").sql("select deptId, count(*) as c from emp group by deptId")
- .returnsUnordered("DEPTID=31; C=1", "DEPTID=33; C=2", "DEPTID=34; C=2", "DEPTID=null; C=1")
- .planContains(CollapsingAggregate.class); // make sure using drill
- }
+// @Test
+// public void testDistinct() throws Exception {
+// JdbcAssert.withModel(MODEL, "HR").sql("select distinct deptId from emp")
+// .returnsUnordered("DEPTID=null", "DEPTID=31", "DEPTID=34", "DEPTID=33")
+// .planContains(CollapsingAggregate.class);
+// }
+//
+// @Test
+// public void testCountNoGroupBy() throws Exception {
+// // 5 out of 6 employees have a not-null deptId
+// JdbcAssert.withModel(MODEL, "HR").sql("select count(deptId) as cd, count(*) as c from emp").returns("CD=5; C=6\n")
+// .planContains(CollapsingAggregate.class);
+// }
+//
+// @Test
+// public void testDistinctCountNoGroupBy() throws Exception {
+// JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp").returns("C=3\n")
+// .planContains(CollapsingAggregate.class);
+// }
+//
+// @Test
+// public void testDistinctCountGroupByEmpty() throws Exception {
+// JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
+// .planContains(CollapsingAggregate.class);
+// }
+//
+// @Test
+// public void testCountNull() throws Exception {
+// JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
+// .planContains(CollapsingAggregate.class);
+// }
+//
+// @Test
+// public void testCount() throws Exception {
+// JdbcAssert.withModel(MODEL, "HR").sql("select deptId, count(*) as c from emp group by deptId")
+// .returnsUnordered("DEPTID=31; C=1", "DEPTID=33; C=2", "DEPTID=34; C=2", "DEPTID=null; C=1")
+// .planContains(CollapsingAggregate.class); // make sure using drill
+// }
@Test
public void testJoin() throws Exception {
[3/5] git commit: Update Hive test data generator so that it won't
fail on subsequent runs.
Posted by ja...@apache.org.
Update Hive test data generator so that it won't fail on subsequent runs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/17ea5432
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/17ea5432
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/17ea5432
Branch: refs/heads/master
Commit: 17ea5432c35ba8dc6858b940fbe349955ecb74c6
Parents: 822bd9c
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Mar 11 08:43:00 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 08:43:00 2014 -0700
----------------------------------------------------------------------
.../exec/store/hive/HiveTestDataGenerator.java | 22 +++++++++++++++++---
1 file changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/17ea5432/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 7155a52..87b14ed 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
@@ -33,18 +34,33 @@ public class HiveTestDataGenerator {
static int RETRIES = 5;
private Driver hiveDriver = null;
-
+ private static final String DB_DIR = "/tmp/drill_hive_db";
+ private static final String WH_DIR = "/tmp/drill_hive_wh";
+
public static void main(String[] args) throws Exception {
HiveTestDataGenerator htd = new HiveTestDataGenerator();
htd.generateTestData();
}
+ private void cleanDir(String dir) throws IOException{
+ File f = new File(dir);
+ if(f.exists()){
+ FileUtils.cleanDirectory(f);
+ FileUtils.forceDelete(f);
+ }
+ }
+
public void generateTestData() throws Exception {
+
+ // remove data from previous runs.
+ cleanDir(DB_DIR);
+ cleanDir(WH_DIR);
+
HiveConf conf = new HiveConf();
- conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=/tmp/drill_hive_db;create=true");
+ conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", DB_DIR));
conf.set("fs.default.name", "file:///");
- conf.set("hive.metastore.warehouse.dir", "/tmp/drill_hive_wh");
+ conf.set("hive.metastore.warehouse.dir", WH_DIR);
String tableName = "kv";