You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/08/29 01:33:50 UTC
[01/47] DRILL-1281: Read into Direct Memory in Parquet Reader.
Requires Hadoop 2.4 or above
Repository: incubator-drill
Updated Branches:
refs/heads/master 654c879f7 -> c1c0eba5b
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f023ef1..a984a4f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,17 @@
<enabled>true</enabled>
</snapshots>
</repository>
+ <repository>
+ <id>mapr-drill-thirdparty</id>
+ <name>MapR Drill Third Party Artifacts</name>
+ <url>http://repository.mapr.com/nexus/content/repositories/drill/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
</repositories>
<issueManagement>
@@ -565,8 +576,8 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>1.2.1</version>
+ <artifactId>hadoop-client</artifactId>
+ <version>2.4.1</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -577,6 +588,14 @@
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>jets3t</artifactId>
<groupId>net.java.dev.jets3t</groupId>
</exclusion>
@@ -584,7 +603,10 @@
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
-
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -623,7 +645,6 @@
</exclusion>
</exclusions>
</dependency>
-
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
@@ -651,6 +672,21 @@
</exclusion>
</exclusions>
</dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>2.4.1</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</dependencyManagement>
</profile>
@@ -663,30 +699,9 @@
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- <version>2.11.0</version>
- </dependency>
- <dependency>
- <groupId>xalan</groupId>
- <artifactId>xalan</artifactId>
- <version>2.7.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>0.94.13-mapr-1401-m7-3.0.2</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>1.0.3-mapr-3.0.2</version>
+ <artifactId>hadoop-client</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -697,35 +712,14 @@
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
- <artifactId>asm</artifactId>
- <groupId>asm</groupId>
- </exclusion>
- <exclusion>
- <artifactId>central-logging</artifactId>
- <groupId>com.mapr.util</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
- <artifactId>mockito-all</artifactId>
- <groupId>org.codehaus</groupId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
</exclusion>
<exclusion>
-
<artifactId>jets3t</artifactId>
<groupId>net.java.dev.jets3t</groupId>
</exclusion>
@@ -733,7 +727,10 @@
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
-
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
<exclusion>
<artifactId>mockito-all</artifactId>
<groupId>org.mockito</groupId>
@@ -747,32 +744,71 @@
<groupId>commons-logging</groupId>
</exclusion>
<exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
+ <artifactId>jersey-server</artifactId>
+ <groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
- <artifactId>servlet-api-2.5</artifactId>
- <groupId>org.mortbay.jetty</groupId>
+ <artifactId>core</artifactId>
+ <groupId>org.eclipse.jdt</groupId>
</exclusion>
<exclusion>
- <artifactId>jasper-runtime</artifactId>
- <groupId>tomcat</groupId>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
- <artifactId>jasper-compiler</artifactId>
- <groupId>tomcat</groupId>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
- <artifactId>jetty</artifactId>
- <groupId>org.mortbay.jetty</groupId>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
</exclusion>
<exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
- <artifactId>core</artifactId>
- <groupId>org.eclipse.jdt</groupId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>2.11.0</version>
+ </dependency>
+ <dependency>
+ <groupId>xalan</groupId>
+ <artifactId>xalan</artifactId>
+ <version>2.7.1</version>
+ </dependency>
+ <dependency>
+ <groupId>sqlline</groupId>
+ <artifactId>sqlline</artifactId>
+ <version>1.1.6</version>
+ </dependency>
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.17-mapr-1405-4.0.0-FCS</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -781,12 +817,12 @@
<repositories>
<repository>
<id>mapr-releases</id>
- <url>http://repository.mapr.com/maven/</url>
+ <url>http://maven.corp.maprtech.com/nexus/content/repositories/public</url>
<snapshots>
- <enabled>false</enabled>
+ <enabled>true</enabled>
</snapshots>
<releases>
- <enabled>true</enabled>
+ <enabled>false</enabled>
</releases>
</repository>
</repositories>
@@ -801,7 +837,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.0.0-cdh4.4.0</version>
+ <version>2.3.0-cdh5.0.3</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
@@ -836,7 +872,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.1.0.2.0.5.0-67</version>
+ <version>2.4.0.2.1.3.0-563</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
[19/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
index fcb3b77..0fa0da2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
@@ -19,6 +19,11 @@ package org.apache.drill.exec.expr.fn.impl;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import java.nio.charset.Charset;
+
+import javax.inject.Inject;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -29,12 +34,11 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
import org.apache.drill.exec.record.RecordBatch;
-import java.nio.charset.Charset;
-import org.apache.drill.exec.expr.holders.IntHolder;
public class StringFunctions{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StringFunctions.class);
@@ -51,14 +55,17 @@ public class StringFunctions{
@Param VarCharHolder input;
@Param(constant=true) VarCharHolder pattern;
@Output BitHolder out;
- @Workspace java.util.regex.Pattern regPattern;
+ @Workspace java.util.regex.Matcher matcher;
public void setup(RecordBatch incoming){
- regPattern = java.util.regex.Pattern.compile(org.apache.drill.exec.expr.fn.impl.RegexpUtil.sqlToRegexLike(pattern.toString()));
+ matcher = java.util.regex.Pattern.compile(org.apache.drill.exec.expr.fn.impl.RegexpUtil.sqlToRegexLike( //
+ org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(pattern.start, pattern.end, pattern.buffer))).matcher("");
}
public void eval(){
- out.value = regPattern.matcher(input.toString()).matches()? 1:0;
+ String i = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+ matcher.reset(i);
+ out.value = matcher.matches()? 1:0;
}
}
@@ -67,14 +74,17 @@ public class StringFunctions{
@Param VarCharHolder input;
@Param(constant=true) VarCharHolder pattern;
@Output BitHolder out;
- @Workspace java.util.regex.Pattern regPattern;
+ @Workspace java.util.regex.Matcher matcher;
public void setup(RecordBatch incoming){
- regPattern = java.util.regex.Pattern.compile(org.apache.drill.exec.expr.fn.impl.RegexpUtil.sqlToRegexSimilar(pattern.toString()));
+
+ matcher = java.util.regex.Pattern.compile(org.apache.drill.exec.expr.fn.impl.RegexpUtil.sqlToRegexSimilar(org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(pattern.start, pattern.end, pattern.buffer))).matcher("");
}
public void eval(){
- out.value = regPattern.matcher(input.toString()).matches()? 1:0;
+ String i = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+ matcher.reset(i);
+ out.value = matcher.matches()? 1:0;
}
}
@@ -87,20 +97,21 @@ public class StringFunctions{
@Param VarCharHolder input;
@Param(constant=true) VarCharHolder pattern;
@Param VarCharHolder replacement;
- @Workspace ByteBuf buffer;
- @Workspace java.util.regex.Pattern regPattern;
+ @Inject DrillBuf buffer;
+ @Workspace java.util.regex.Matcher matcher;
@Output VarCharHolder out;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
- regPattern = java.util.regex.Pattern.compile(pattern.toString());
+ matcher = java.util.regex.Pattern.compile(org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(pattern.start, pattern.end, pattern.buffer)).matcher("");
}
public void eval(){
- out.buffer = buffer;
- out.start = 0;
- byte [] bytea = regPattern.matcher(input.toString()).replaceAll(replacement.toString()).getBytes(java.nio.charset.Charset.forName("UTF-8"));
+ out.start = 0;
+ String i = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+ String r = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(replacement.start, replacement.end, replacement.buffer);
+ byte [] bytea = matcher.reset(i).replaceAll(r).getBytes(java.nio.charset.Charset.forName("UTF-8"));
+ out.buffer = buffer = buffer.reallocIfNeeded(bytea.length);
out.buffer.setBytes(out.start, bytea);
out.end = bytea.length;
}
@@ -224,14 +235,13 @@ public class StringFunctions{
@Param VarCharHolder input;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval(){
- out.buffer = buffer;
+ out.buffer = buffer = buffer.reallocIfNeeded(input.end- input.start);
out.start = 0;
out.end = input.end - input.start;
@@ -256,14 +266,13 @@ public class StringFunctions{
@Param VarCharHolder input;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval() {
- out.buffer = buffer;
+ out.buffer = buffer = buffer.reallocIfNeeded(input.end- input.start);
out.start = 0;
out.end = input.end - input.start;
@@ -446,51 +455,16 @@ public class StringFunctions{
@Param VarCharHolder input;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval() {
- out.buffer = buffer;
+ out.buffer = buffer = buffer.reallocIfNeeded(input.end - input.start);
out.start = 0;
out.end = input.end - input.start;
-
- // Assumes Alpha as [A-Za-z0-9]
- // white space is treated as everything else.
- boolean capNext = true;
- for (int id = input.start; id < input.end; id++) {
- byte currentByte = input.buffer.getByte(id);
-
- // 'A - Z' : 0x41 - 0x5A
- // 'a - z' : 0x61 - 0x7A
- // '0-9' : 0x30 - 0x39
- if (capNext) { // curCh is whitespace or first character of word.
- if (currentByte >= 0x30 && currentByte <= 0x39) { // 0-9
- capNext = false;
- } else if (currentByte >=0x41 && currentByte <= 0x5A) { // A-Z
- capNext = false;
- } else if (currentByte >= 0x61 && currentByte <= 0x7A) { // a-z
- capNext = false;
- currentByte -= 0x20; // Uppercase this character
- }
- // else {} whitespace
- } else { // Inside of a word or white space after end of word.
- if (currentByte >= 0x30 && currentByte <= 0x39) { // 0-9
- // noop
- } else if (currentByte >=0x41 && currentByte <= 0x5A) { // A-Z
- currentByte -= 0x20 ; // Lowercase this character
- } else if (currentByte >= 0x61 && currentByte <= 0x7A) { // a-z
- // noop
- } else { // whitespace
- capNext = true;
- }
- }
-
- out.buffer.setByte(id - input.start, currentByte) ;
- } //end of for_loop
-
+ org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.initCap(input.start, input.end, input.buffer, out.buffer);
}
}
@@ -502,11 +476,11 @@ public class StringFunctions{
@Param VarCharHolder text;
@Param VarCharHolder from;
@Param VarCharHolder to;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output VarCharHolder out;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
+ buffer = buffer.reallocIfNeeded(8000);
}
public void eval(){
@@ -566,15 +540,15 @@ public class StringFunctions{
@Param VarCharHolder text;
@Param BigIntHolder length;
@Param VarCharHolder fill;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output VarCharHolder out;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval() {
+ buffer = buffer.reallocIfNeeded((int) length.value*2);
byte currentByte = 0;
int id = 0;
//get the char length of text.
@@ -637,15 +611,16 @@ public class StringFunctions{
@Param VarCharHolder text;
@Param BigIntHolder length;
@Param VarCharHolder fill;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output VarCharHolder out;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval() {
+ buffer = buffer.reallocIfNeeded((int) length.value*2);
+
byte currentByte = 0;
int id = 0;
//get the char length of text.
@@ -771,10 +746,10 @@ public class StringFunctions{
*/
@FunctionTemplate(name = "btrim", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Btrim implements DrillSimpleFunc{
-
+
@Param VarCharHolder text;
@Param VarCharHolder from;
-
+
@Output VarCharHolder out;
public void setup(RecordBatch incoming){
@@ -784,14 +759,14 @@ public class StringFunctions{
out.buffer = text.buffer;
out.start = out.end = text.start;
int bytePerChar = 0;
-
+
//Scan from left of "text", stop until find a char not in "from"
for (int id = text.start; id < text.end; id += bytePerChar) {
bytePerChar = org.apache.drill.exec.expr.fn.impl.StringFunctionUtil.utf8CharLen(text.buffer, id);
int pos = org.apache.drill.exec.expr.fn.impl.StringFunctionUtil.stringLeftMatchUTF8(from.buffer, from.start, from.end,
text.buffer, id, id + bytePerChar);
if (pos < 0) { // Found the 1st char not in "from", stop
- out.start = id;
+ out.start = id;
break;
}
}
@@ -803,7 +778,7 @@ public class StringFunctions{
int pos = org.apache.drill.exec.expr.fn.impl.StringFunctionUtil.stringLeftMatchUTF8(from.buffer, from.start, from.end,
text.buffer, id, id + bytePerChar);
if (pos < 0) { // Found the 1st char not in "from", stop
- out.end = id+ bytePerChar;
+ out.end = id+ bytePerChar;
break;
}
}
@@ -818,15 +793,14 @@ public class StringFunctions{
@Param VarCharHolder left;
@Param VarCharHolder right;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval(){
- out.buffer = buffer;
+ out.buffer = buffer = buffer.reallocIfNeeded( (left.end - left.start) + (right.end - right.start));
out.start = out.end = 0;
int id = 0;
@@ -845,15 +819,14 @@ public class StringFunctions{
@Param VarCharHolder left;
@Param NullableVarCharHolder right;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval(){
- out.buffer = buffer;
+ out.buffer = buffer = buffer.reallocIfNeeded( (left.end - left.start) + (right.end - right.start));;
out.start = out.end = 0;
int id = 0;
@@ -873,15 +846,14 @@ public class StringFunctions{
@Param NullableVarCharHolder left;
@Param VarCharHolder right;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval(){
- out.buffer = buffer;
+ out.buffer = buffer.reallocIfNeeded( (left.end - left.start) + (right.end - right.start));
out.start = out.end = 0;
int id = 0;
@@ -901,15 +873,14 @@ public class StringFunctions{
@Param NullableVarCharHolder left;
@Param NullableVarCharHolder right;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [8000]);
}
public void eval(){
- out.buffer = buffer;
+ out.buffer = buffer.reallocIfNeeded( (left.end - left.start) + (right.end - right.start));
out.start = out.end = 0;
int id = 0;
@@ -969,14 +940,14 @@ public class StringFunctions{
@Param IntHolder in;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buf;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [1]);
+ buf = buf.reallocIfNeeded(1);
}
public void eval() {
- out.buffer = buffer;
+ out.buffer = buf;
out.start = out.end = 0;
out.buffer.setByte(0, in.value);
++out.end;
@@ -992,24 +963,22 @@ public class StringFunctions{
@Param VarCharHolder in;
@Param IntHolder nTimes;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming) {
}
public void eval() {
+ final int len = in.end - in.start;
int num = nTimes.value;
- byte[] bytea = new byte [(in.end - in.start)*num];
- int index = 0;
- while(num > 0){
- for (int id = in.start; id < in.end; id++){
- bytea[index++] = in.buffer.getByte(id);
- }
- num--;
- }
- out.buffer = io.netty.buffer.Unpooled.wrappedBuffer(bytea);
+ System.out.println(len + ":" + num);
out.start = 0;
- out.end = bytea.length;
+ out.buffer = buffer = buffer.reallocIfNeeded( len * num );
+ for(int i =0; i < num; i++){
+ in.buffer.getBytes(in.start, out.buffer, i * len, len);
+ }
+ out.end = len * num;
+ System.out.println(out.end);
}
}
@@ -1023,9 +992,10 @@ public class StringFunctions{
@Param VarCharHolder enc;
@Output VarCharHolder out;
@Workspace Charset inCharset;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming) {
- inCharset = java.nio.charset.Charset.forName(enc.toString());
+ inCharset = java.nio.charset.Charset.forName(org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(enc.start, enc.end, enc.buffer));
}
public void eval() {
@@ -1035,7 +1005,8 @@ public class StringFunctions{
bytea[index]=in.buffer.getByte(i);
}
byte[] outBytea = new String(bytea, inCharset).getBytes(com.google.common.base.Charsets.UTF_8);
- out.buffer = io.netty.buffer.Unpooled.wrappedBuffer(outBytea);
+ out.buffer = buffer = buffer.reallocIfNeeded(outBytea.length);
+ out.buffer.setBytes(0, outBytea);
out.start = 0;
out.end = outBytea.length;
}
@@ -1049,31 +1020,31 @@ public class StringFunctions{
@Param VarCharHolder in;
@Output VarCharHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming) {
}
public void eval() {
- int charlen = 0;
+ final int len = in.end - in.start;
+ out.start = 0;
+ out.end = len;
+ out.buffer = buffer = buffer.reallocIfNeeded(len);
+ int charlen = 0;
- byte[] bytea = new byte [in.end - in.start];
- int index = in.end;
- int innerindex = 0;
+ int index = in.end;
+ int innerindex = 0;
- for (int id = in.start; id < in.end; id+=charlen){
- innerindex = charlen = org.apache.drill.exec.expr.fn.impl.StringFunctionUtil.utf8CharLen(in.buffer, id);
+ for (int id = in.start; id < in.end; id+=charlen){
+ innerindex = charlen = org.apache.drill.exec.expr.fn.impl.StringFunctionUtil.utf8CharLen(in.buffer, id);
while(innerindex > 0){
- bytea[index - innerindex] = in.buffer.getByte(id + (charlen - innerindex));
+ out.buffer.setByte(index - innerindex, in.buffer.getByte(id + (charlen - innerindex)));
innerindex-- ;
}
index -= charlen;
- }
- out.buffer = io.netty.buffer.Unpooled.wrappedBuffer(bytea);
- out.start = 0;
- out.end = bytea.length;
}
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntBEConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntBEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntBEConvertTo.java
index 696ea25..e645f4b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntBEConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntBEConvertTo.java
@@ -17,7 +17,9 @@
*/
package org.apache.drill.exec.expr.fn.impl.conv;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import javax.inject.Inject;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -25,7 +27,6 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
import org.apache.drill.exec.record.RecordBatch;
@@ -35,11 +36,11 @@ public class BigIntBEConvertTo implements DrillSimpleFunc {
@Param BigIntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntConvertTo.java
index 8c76113..7f6c8d2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class BigIntConvertTo implements DrillSimpleFunc {
@Param BigIntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntVLongConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntVLongConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntVLongConvertTo.java
index 78564f6..7a6f302 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntVLongConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BigIntVLongConvertTo.java
@@ -17,7 +17,10 @@
*/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -35,14 +38,15 @@ public class BigIntVLongConvertTo implements DrillSimpleFunc {
@Param BigIntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
/* Hadoop Variable length integer (represented in the same way as a long)
* occupies between 1-9 bytes.
*/
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(9);
+ buffer = buffer.reallocIfNeeded(9);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BooleanByteConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BooleanByteConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BooleanByteConvertTo.java
index 0feb08b..c2a63b5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BooleanByteConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/BooleanByteConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class BooleanByteConvertTo implements DrillSimpleFunc {
@Param BitHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(1);
+ buffer = buffer.reallocIfNeeded(1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochBEConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochBEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochBEConvertTo.java
index 9e2026a..4affa55 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochBEConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochBEConvertTo.java
@@ -17,7 +17,10 @@
*/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -35,11 +38,12 @@ public class DateEpochBEConvertTo implements DrillSimpleFunc {
@Param DateHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochConvertTo.java
index 90fb319..adaba9f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DateEpochConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class DateEpochConvertTo implements DrillSimpleFunc {
@Param DateHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DoubleConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DoubleConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DoubleConvertTo.java
index 5487ee0..dd0c7f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DoubleConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/DoubleConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class DoubleConvertTo implements DrillSimpleFunc {
@Param Float8Holder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/FloatConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/FloatConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/FloatConvertTo.java
index ec06225..9175901 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/FloatConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/FloatConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class FloatConvertTo implements DrillSimpleFunc {
@Param Float4Holder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(4);
+ buffer = buffer.reallocIfNeeded(4);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntBEConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntBEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntBEConvertTo.java
index 6de4f0a..45687f5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntBEConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntBEConvertTo.java
@@ -17,7 +17,10 @@
*/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -35,11 +38,12 @@ public class IntBEConvertTo implements DrillSimpleFunc {
@Param IntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(4);
+ buffer = buffer.reallocIfNeeded(4);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntConvertTo.java
index c1f8977..0ef7700 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class IntConvertTo implements DrillSimpleFunc {
@Param IntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(4);
+ buffer = buffer.reallocIfNeeded(4);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntVIntConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntVIntConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntVIntConvertTo.java
index 52680c8..be09245 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntVIntConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/IntVIntConvertTo.java
@@ -17,7 +17,10 @@
*/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -35,14 +38,15 @@ public class IntVIntConvertTo implements DrillSimpleFunc {
@Param IntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
/* Hadoop Variable length integer (represented in the same way as a long)
* occupies between 1-9 bytes.
*/
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(9);
+ buffer = buffer.reallocIfNeeded(9);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
index d8df543..53b8f50 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
@@ -22,7 +22,10 @@ package org.apache.drill.exec.expr.fn.impl.conv;
import java.io.ByteArrayOutputStream;
import java.io.StringReader;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -50,6 +53,8 @@ public class JsonConvertFrom {
public static class ConvertFromJson implements DrillSimpleFunc{
@Param VarBinaryHolder in;
+ @Inject DrillBuf buffer;
+
@Output ComplexWriter writer;
public void setup(RecordBatch incoming){
@@ -60,23 +65,24 @@ public class JsonConvertFrom {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
-
+
try {
- org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader();
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer);
jsonReader.write(new java.io.StringReader(input), writer);
-
+
} catch (Exception e) {
System.out.println(" msg = " + e.getMessage() + " trace : " + e.getStackTrace());
}
}
}
-
+
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL, isRandom = true)
public static class ConvertFromJsonVarchar implements DrillSimpleFunc{
@Param VarCharHolder in;
@Output ComplexWriter writer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
}
@@ -86,12 +92,12 @@ public class JsonConvertFrom {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
-
+
try {
- org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader();
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer);
jsonReader.write(new java.io.StringReader(input), writer);
-
+
} catch (Exception e) {
System.out.println(" msg = " + e.getMessage() + " trace : " + e.getStackTrace());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
index 8b0e811..c87bd5c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
@@ -18,25 +18,20 @@
package org.apache.drill.exec.expr.fn.impl.conv;
-import java.io.ByteArrayOutputStream;
+import io.netty.buffer.DrillBuf;
-import io.netty.buffer.ByteBuf;
+import javax.inject.Inject;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
-import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.complex.fn.JsonWriter;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import com.google.common.base.Charsets;
-
public class JsonConvertTo {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonConvertTo.class);
@@ -48,14 +43,12 @@ public class JsonConvertTo {
@Param FieldReader input;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch incoming){
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(256);
}
public void eval(){
- out.buffer = buffer;
out.start = 0;
java.io.ByteArrayOutputStream stream = new java.io.ByteArrayOutputStream();
@@ -69,12 +62,8 @@ public class JsonConvertTo {
byte [] bytea = stream.toByteArray();
- if (bytea.length > buffer.capacity()) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(bytea.length);
- out.buffer = buffer;
- }
-
- out.buffer.setBytes(out.start, bytea);
+ out.buffer = buffer = buffer.reallocIfNeeded(bytea.length);
+ out.buffer.setBytes(0, bytea);
out.end = bytea.length;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntBEConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntBEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntBEConvertTo.java
index 227932a..68c9f33 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntBEConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntBEConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class SmallIntBEConvertTo implements DrillSimpleFunc {
@Param SmallIntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(2);
+ buffer = buffer.reallocIfNeeded(2);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntConvertTo.java
index db74fad..2d97033 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/SmallIntConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class SmallIntConvertTo implements DrillSimpleFunc {
@Param SmallIntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(2);
+ buffer = buffer.reallocIfNeeded(2);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochBEConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochBEConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochBEConvertTo.java
index e19850e..06cd085 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochBEConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochBEConvertTo.java
@@ -17,7 +17,10 @@
*/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -35,11 +38,12 @@ public class TimeEpochBEConvertTo implements DrillSimpleFunc {
@Param TimeHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochConvertTo.java
index 56b76e0..d3802e8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TimeEpochConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class TimeEpochConvertTo implements DrillSimpleFunc {
@Param TimeHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TinyIntConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TinyIntConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TinyIntConvertTo.java
index afd1eb6..f4d582e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TinyIntConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/TinyIntConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -37,11 +40,12 @@ public class TinyIntConvertTo implements DrillSimpleFunc {
@Param TinyIntHolder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(1);
+ buffer = buffer.reallocIfNeeded(1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt8ConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt8ConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt8ConvertTo.java
index 0322d1a..4e6d606 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt8ConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/UInt8ConvertTo.java
@@ -18,7 +18,10 @@
******************************************************************************/
package org.apache.drill.exec.expr.fn.impl.conv;
+import javax.inject.Inject;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -36,11 +39,12 @@ public class UInt8ConvertTo implements DrillSimpleFunc {
@Param UInt8Holder in;
@Output VarBinaryHolder out;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
+
@Override
public void setup(RecordBatch incoming) {
- buffer = org.apache.drill.exec.util.ByteBufUtil.createBuffer(8);
+ buffer = buffer.reallocIfNeeded(8);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
index cd08367..7cf445c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.memory;
-import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.ByteBuf;
import java.util.Arrays;
@@ -58,7 +58,7 @@ public class Accountor {
}
}
- public boolean transferTo(Accountor target, AccountingByteBuf buf, long size){
+ public boolean transferTo(Accountor target, DrillBuf buf, long size){
boolean withinLimit = target.forceAdditionalReservation(size);
release(buf, size);
@@ -96,7 +96,7 @@ public class Accountor {
}
}
- public void reserved(long expected, AccountingByteBuf buf){
+ public void reserved(long expected, DrillBuf buf){
// make sure to take away the additional memory that happened due to rounding.
long additional = buf.capacity() - expected;
@@ -108,7 +108,7 @@ public class Accountor {
}
- public void releasePartial(AccountingByteBuf buf, long size){
+ public void releasePartial(DrillBuf buf, long size){
remainder.returnAllocation(size);
if (ENABLE_ACCOUNTING) {
if(buf != null){
@@ -122,7 +122,7 @@ public class Accountor {
}
}
- public void release(AccountingByteBuf buf, long size) {
+ public void release(DrillBuf buf, long size) {
remainder.returnAllocation(size);
if (ENABLE_ACCOUNTING) {
if(buf != null && buffers.remove(buf) == null) throw new IllegalStateException("Releasing a buffer that has already been released. Buffer: " + buf);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
index 38fe4f1..67a041d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.memory;
-import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.ByteBufAllocator;
import java.io.Closeable;
@@ -38,7 +38,7 @@ public interface BufferAllocator extends Closeable {
* The size in bytes.
* @return A new ByteBuf.
*/
- public abstract AccountingByteBuf buffer(int size);
+ public abstract DrillBuf buffer(int size);
/**
* Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
@@ -48,7 +48,7 @@ public interface BufferAllocator extends Closeable {
* @param maxSize The maximum size in bytes.
* @return A new ByteBuf.
*/
- public abstract AccountingByteBuf buffer(int minSize, int maxSize);
+ public abstract DrillBuf buffer(int minSize, int maxSize);
public abstract ByteBufAllocator getUnderlyingAllocator();
@@ -60,7 +60,7 @@ public interface BufferAllocator extends Closeable {
* @param buf
* @return false if over allocation.
*/
- public boolean takeOwnership(AccountingByteBuf buf) ;
+ public boolean takeOwnership(DrillBuf buf) ;
public PreAllocator getNewPreAllocator();
@@ -71,7 +71,7 @@ public interface BufferAllocator extends Closeable {
public interface PreAllocator {
public boolean preAllocate(int bytes);
- public AccountingByteBuf getAllocation();
+ public DrillBuf getAllocation();
}
/**
@@ -91,4 +91,5 @@ public interface BufferAllocator extends Closeable {
public abstract long getAllocatedMemory();
+ public DrillBuf getEmpty();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index d89c892..55f11a4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -17,11 +17,12 @@
*/
package org.apache.drill.exec.memory;
-import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocatorL;
+import io.netty.buffer.UnsafeDirectLittleEndian;
import java.util.HashMap;
import java.util.IdentityHashMap;
@@ -38,9 +39,11 @@ public class TopLevelAllocator implements BufferAllocator {
private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
private final Map<ChildAllocator, StackTraceElement[]> childrenMap;
- private final PooledByteBufAllocator innerAllocator = PooledByteBufAllocatorL.DEFAULT;
+ private final PooledByteBufAllocatorL innerAllocator = PooledByteBufAllocatorL.DEFAULT;
private final Accountor acct;
private final boolean errorOnLeak;
+ private final DrillBuf empty;
+
@Deprecated
public TopLevelAllocator() {
@@ -55,6 +58,7 @@ public class TopLevelAllocator implements BufferAllocator {
private TopLevelAllocator(long maximumAllocation, boolean errorOnLeak){
this.errorOnLeak = errorOnLeak;
this.acct = new Accountor(errorOnLeak, null, null, maximumAllocation, 0);
+ this.empty = DrillBuf.getEmpty(this, acct);
this.childrenMap = ENABLE_ACCOUNTING ? new IdentityHashMap<ChildAllocator, StackTraceElement[]>() : null;
}
@@ -65,20 +69,21 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
- public boolean takeOwnership(AccountingByteBuf buf) {
+ public boolean takeOwnership(DrillBuf buf) {
return buf.transferAccounting(acct);
}
- public AccountingByteBuf buffer(int min, int max) {
+ public DrillBuf buffer(int min, int max) {
+ if(min == 0) return empty;
if(!acct.reserve(min)) return null;
- ByteBuf buffer = innerAllocator.directBuffer(min, max);
- AccountingByteBuf wrapped = new AccountingByteBuf(acct, buffer);
+ UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(min, max);
+ DrillBuf wrapped = new DrillBuf(this, acct, buffer);
acct.reserved(min, wrapped);
return wrapped;
}
@Override
- public AccountingByteBuf buffer(int size) {
+ public DrillBuf buffer(int size) {
return buffer(size, size);
}
@@ -124,8 +129,16 @@ public class TopLevelAllocator implements BufferAllocator {
}
- private class ChildAllocator implements BufferAllocator{
+ @Override
+ public DrillBuf getEmpty() {
+ return empty;
+ }
+
+
+
+ private class ChildAllocator implements BufferAllocator{
+ private final DrillBuf empty;
private Accountor childAcct;
private Map<ChildAllocator, StackTraceElement[]> children = new HashMap<>();
private boolean closed = false;
@@ -137,27 +150,29 @@ public class TopLevelAllocator implements BufferAllocator {
childAcct = new Accountor(errorOnLeak, handle, parentAccountor, max, pre);
this.handle = handle;
thisMap = map;
+ this.empty = DrillBuf.getEmpty(this, childAcct);
}
@Override
- public boolean takeOwnership(AccountingByteBuf buf) {
+ public boolean takeOwnership(DrillBuf buf) {
return buf.transferAccounting(childAcct);
}
@Override
- public AccountingByteBuf buffer(int size, int max) {
+ public DrillBuf buffer(int size, int max) {
+ if(size == 0) return empty;
if(!childAcct.reserve(size)){
logger.warn("Unable to allocate buffer of size {} due to memory limit. Current allocation: {}", size, getAllocatedMemory());
return null;
};
- ByteBuf buffer = innerAllocator.directBuffer(size, max);
- AccountingByteBuf wrapped = new AccountingByteBuf(childAcct, buffer);
+ UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(size, max);
+ DrillBuf wrapped = new DrillBuf(this, childAcct, buffer);
childAcct.reserved(buffer.capacity(), wrapped);
return wrapped;
}
- public AccountingByteBuf buffer(int size) {
+ public DrillBuf buffer(int size) {
return buffer(size, size);
}
@@ -179,7 +194,7 @@ public class TopLevelAllocator implements BufferAllocator {
}
public PreAllocator getNewPreAllocator(){
- return new PreAlloc(this.childAcct);
+ return new PreAlloc(this, this.childAcct);
}
@Override
@@ -221,17 +236,25 @@ public class TopLevelAllocator implements BufferAllocator {
return childAcct.getAllocation();
}
+ @Override
+ public DrillBuf getEmpty() {
+ return empty;
+ }
+
+
}
public PreAllocator getNewPreAllocator(){
- return new PreAlloc(this.acct);
+ return new PreAlloc(this, this.acct);
}
public class PreAlloc implements PreAllocator{
int bytes = 0;
final Accountor acct;
- private PreAlloc(Accountor acct){
+ final BufferAllocator allocator;
+ private PreAlloc(BufferAllocator allocator, Accountor acct){
this.acct = acct;
+ this.allocator = allocator;
}
/**
@@ -248,8 +271,8 @@ public class TopLevelAllocator implements BufferAllocator {
}
- public AccountingByteBuf getAllocation(){
- AccountingByteBuf b = new AccountingByteBuf(acct, innerAllocator.buffer(bytes));
+ public DrillBuf getAllocation(){
+ DrillBuf b = new DrillBuf(allocator, acct, innerAllocator.directBuffer(bytes, bytes));
acct.reserved(bytes, b);
return b;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 6fccb3b..6bdce3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -17,8 +17,13 @@
*/
package org.apache.drill.exec.ops;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import io.netty.buffer.DrillBuf;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.jdbc.SimpleOptiqSchema;
import org.apache.drill.common.config.DrillConfig;
@@ -47,6 +52,9 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import com.carrotsearch.hppc.LongObjectOpenHashMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* Contextual objects required for execution of a particular fragment.
@@ -70,6 +78,7 @@ public class FragmentContext implements Closeable {
private final long queryStartTime;
private final int rootFragmentTimeZone;
private final OptionManager fragmentOptions;
+ private LongObjectOpenHashMap<DrillBuf> managedBuffers = new LongObjectOpenHashMap<>();
private volatile Throwable failureCause;
private volatile boolean failed = false;
@@ -77,7 +86,7 @@ public class FragmentContext implements Closeable {
public FragmentContext(DrillbitContext dbContext, PlanFragment fragment, UserClientConnection connection,
FunctionImplementationRegistry funcRegistry) throws OutOfMemoryException, ExecutionSetupException {
- this.transformer = new ClassTransformer();
+ this.transformer = new ClassTransformer(dbContext.getCache());
this.stats = new FragmentStats(dbContext.getMetrics());
this.context = dbContext;
this.connection = connection;
@@ -182,7 +191,7 @@ public class FragmentContext implements Closeable {
}
public <T> T getImplementationClass(CodeGenerator<T> cg) throws ClassTransformationException, IOException {
- return transformer.getImplementationClass(this.loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName());
+ return context.getCompiler().getImplementationClass(cg);
}
/**
@@ -251,10 +260,32 @@ public class FragmentContext implements Closeable {
for(Thread thread: daemonThreads){
thread.interrupt();
}
+ Object[] mbuffers = ((LongObjectOpenHashMap<Object>)(Object)managedBuffers).values;
+ for(int i =0; i < mbuffers.length; i++){
+ if(managedBuffers.allocated[i]) ((DrillBuf)mbuffers[i]).release();
+ }
+
if (buffers != null) {
buffers.close();
}
allocator.close();
}
+ public DrillBuf replace(DrillBuf old, int newSize){
+ if(managedBuffers.remove(old.memoryAddress()) == null) throw new IllegalStateException("Tried to remove unmanaged buffer.");
+ old.release();
+ return getManagedBuffer(newSize);
+ }
+
+ public DrillBuf getManagedBuffer(){
+ return getManagedBuffer(256);
+ }
+
+ public DrillBuf getManagedBuffer(int size){
+ DrillBuf newBuf = allocator.buffer(size);
+ managedBuffers.put(newBuf.memoryAddress(), newBuf);
+ newBuf.setFragmentContext(this);
+ return newBuf;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
index 2d46733..c5dea4f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContext.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.ops;
+import io.netty.buffer.DrillBuf;
+
import java.util.Iterator;
import org.apache.drill.common.util.Hook.Closeable;
@@ -24,6 +26,8 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.physical.base.PhysicalOperator;
+import com.carrotsearch.hppc.LongObjectOpenHashMap;
+
public class OperatorContext implements Closeable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorContext.class);
@@ -31,6 +35,7 @@ public class OperatorContext implements Closeable {
private boolean closed = false;
private PhysicalOperator popConfig;
private OperatorStats stats;
+ private LongObjectOpenHashMap<DrillBuf> managedBuffers = new LongObjectOpenHashMap<>();
public OperatorContext(PhysicalOperator popConfig, FragmentContext context) throws OutOfMemoryException {
this.allocator = context.getNewChildAllocator(popConfig.getInitialAllocation(), popConfig.getMaxAllocation());
@@ -46,6 +51,23 @@ public class OperatorContext implements Closeable {
this.stats = stats;
}
+ public DrillBuf replace(DrillBuf old, int newSize){
+ if(managedBuffers.remove(old.memoryAddress()) == null) throw new IllegalStateException("Tried to remove unmanaged buffer.");
+ old.release();
+ return getManagedBuffer(newSize);
+ }
+
+ public DrillBuf getManagedBuffer(){
+ return getManagedBuffer(256);
+ }
+
+ public DrillBuf getManagedBuffer(int size){
+ DrillBuf newBuf = allocator.buffer(size);
+ managedBuffers.put(newBuf.memoryAddress(), newBuf);
+ newBuf.setOperatorContext(this);
+ return newBuf;
+ }
+
public static int getChildCount(PhysicalOperator popConfig){
Iterator<PhysicalOperator> iter = popConfig.iterator();
int i = 0;
@@ -76,6 +98,13 @@ public class OperatorContext implements Closeable {
return;
}
logger.debug("Closing context for {}", popConfig != null ? popConfig.getClass().getName() : null);
+
+ // release managed buffers.
+ Object[] buffers = ((LongObjectOpenHashMap<Object>)(Object)managedBuffers).values;
+ for(int i =0; i < buffers.length; i++){
+ if(managedBuffers.allocated[i]) ((DrillBuf)buffers[i]).release();
+ }
+
if (allocator != null) {
allocator.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
index 1aec625..32aae07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
@@ -17,13 +17,14 @@
*/
package org.apache.drill.exec.physical.impl;
+import io.netty.buffer.DrillBuf;
+
+import java.util.List;
+
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.ValueVector;
-import java.util.List;
-
public interface OutputMutator {
public <T extends ValueVector> T addField(MaterializedField field, Class<T> clazz) throws SchemaChangeException ;
public void allocate(int recordCount);
@@ -34,4 +35,5 @@ public interface OutputMutator {
* One to many layers to rip out, address it as a separate JIRA.
*/
public void addFields(List<ValueVector> vvList);
+ public DrillBuf getManagedBuffer();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index e999c9a..6216305 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.physical.impl;
-import io.netty.buffer.Unpooled;
+import io.netty.buffer.DrillBuf;
import java.util.Collections;
import java.util.Iterator;
@@ -47,17 +47,12 @@ import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.util.BatchPrinter;
-import org.apache.drill.exec.util.VectorUtil;
import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.NullableVarCharVector;
import org.apache.drill.exec.vector.ValueVector;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.drill.exec.vector.complex.MapVector;
-import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
/**
* Record batch used for a particular scan. Operators against one or more
@@ -212,6 +207,11 @@ public class ScanBatch implements RecordBatch {
private void addPartitionVectors() throws ExecutionSetupException{
try {
+ if(partitionVectors != null){
+ for(ValueVector v : partitionVectors){
+ v.clear();
+ }
+ }
partitionVectors = Lists.newArrayList();
for (int i : selectedPartitionColumns) {
MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(partitionColumnDesignator + i), Types.optional(MinorType.VARCHAR));
@@ -230,15 +230,9 @@ public class ScanBatch implements RecordBatch {
if (partitionValues.length > i) {
String val = partitionValues[i];
AllocationHelper.allocate(v, recordCount, val.length());
- NullableVarCharHolder h = new NullableVarCharHolder();
byte[] bytes = val.getBytes();
- h.buffer = Unpooled.buffer(bytes.length);
- h.buffer.writeBytes(bytes);
- h.start = 0;
- h.isSet = 1;
- h.end = bytes.length;
for (int j = 0; j < recordCount; j++) {
- v.getMutator().setSafe(j, h);
+ v.getMutator().setSafe(j, bytes, 0, bytes.length);
}
v.getMutator().setValueCount(recordCount);
} else {
@@ -318,6 +312,11 @@ public class ScanBatch implements RecordBatch {
}
return false;
}
+
+ @Override
+ public DrillBuf getManagedBuffer() {
+ return oContext.getManagedBuffer();
+ }
}
@Override
@@ -332,6 +331,9 @@ public class ScanBatch implements RecordBatch {
public void cleanup(){
container.clear();
+ for(ValueVector v : partitionVectors){
+ v.clear();
+ }
fieldVectorMap.clear();
oContext.close();
}
@@ -340,5 +342,5 @@ public class ScanBatch implements RecordBatch {
public VectorContainer getOutgoingContainer() {
throw new UnsupportedOperationException(String.format(" You should not call getOutgoingContainer() for class %s", this.getClass().getCanonicalName()));
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index eca42c9..22df5f6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -42,6 +42,7 @@ import org.apache.drill.exec.physical.impl.common.ChainedHashTable;
import org.apache.drill.exec.physical.impl.common.HashTable;
import org.apache.drill.exec.physical.impl.common.HashTableConfig;
import org.apache.drill.exec.physical.impl.common.HashTableStats;
+import org.apache.drill.exec.physical.impl.common.IndexPointer;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
@@ -83,9 +84,9 @@ public abstract class HashAggTemplate implements HashAggregator {
private HashAggregate hashAggrConfig;
private HashTable htable;
private ArrayList<BatchHolder> batchHolders;
- private IntHolder htIdxHolder; // holder for the Hashtable's internal index returned by put()
- private IntHolder outStartIdxHolder;
- private IntHolder outNumRecordsHolder;
+ private IndexPointer htIdxHolder; // holder for the Hashtable's internal index returned by put()
+ private IndexPointer outStartIdxHolder;
+ private IndexPointer outNumRecordsHolder;
private int numGroupByOutFields = 0; // Note: this should be <= number of group-by fields
List<VectorAllocator> wsAllocators = Lists.newArrayList(); // allocators for the workspace vectors
@@ -97,14 +98,14 @@ public abstract class HashAggTemplate implements HashAggregator {
private OperatorStats stats = null;
private HashTableStats htStats = new HashTableStats();
-
+
public enum Metric implements MetricDef {
NUM_BUCKETS,
NUM_ENTRIES,
NUM_RESIZING,
RESIZING_TIME;
-
+
// duplicate for hash ag
@Override
@@ -150,7 +151,7 @@ public abstract class HashAggTemplate implements HashAggregator {
setupInterior(incoming, outgoing, aggrValuesContainer);
}
- private boolean outputValues(IntHolder outStartIdxHolder, IntHolder outNumRecordsHolder) {
+ private boolean outputValues(IndexPointer outStartIdxHolder, IndexPointer outNumRecordsHolder) {
outStartIdxHolder.value = batchOutputCount;
outNumRecordsHolder.value = 0;
boolean status = true;
@@ -184,7 +185,7 @@ public abstract class HashAggTemplate implements HashAggregator {
private int getNumPendingOutput() {
return getNumGroups() - batchOutputCount;
}
-
+
// Code-generated methods (implemented in HashAggBatch)
@RuntimeOverridden
@@ -199,14 +200,14 @@ public abstract class HashAggTemplate implements HashAggregator {
@Override
- public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig,
- FragmentContext context,
+ public void setup(HashAggregate hashAggrConfig, HashTableConfig htConfig,
+ FragmentContext context,
OperatorStats stats,
BufferAllocator allocator, RecordBatch incoming, HashAggBatch outgoing,
LogicalExpression[] valueExprs,
List<TypedFieldId> valueFieldIds,
- TypedFieldId[] groupByOutFieldIds,
- VectorContainer outContainer)
+ TypedFieldId[] groupByOutFieldIds,
+ VectorContainer outContainer)
throws SchemaChangeException, ClassTransformationException, IOException {
if (valueExprs == null || valueFieldIds == null) {
@@ -236,10 +237,10 @@ public abstract class HashAggTemplate implements HashAggregator {
throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by expressions.");
}
- this.htIdxHolder = new IntHolder();
- this.outStartIdxHolder = new IntHolder();
- this.outNumRecordsHolder = new IntHolder();
-
+ this.htIdxHolder = new IndexPointer();
+ this.outStartIdxHolder = new IndexPointer();
+ this.outNumRecordsHolder = new IndexPointer();
+
materializedValueFields = new MaterializedField[valueFieldIds.size()];
if (valueFieldIds.size() > 0) {
@@ -317,7 +318,7 @@ public abstract class HashAggTemplate implements HashAggregator {
// outcome = out;
buildComplete = true;
-
+
updateStats(htable);
// output the first batch; remaining batches will be output
@@ -426,7 +427,7 @@ public abstract class HashAggTemplate implements HashAggregator {
bh.setup();
}
-
+
public IterOutcome outputCurrentBatch() {
if (outBatchIndex >= batchHolders.size()) {
this.outcome = IterOutcome.NONE;
@@ -435,7 +436,7 @@ public abstract class HashAggTemplate implements HashAggregator {
// get the number of records in the batch holder that are pending output
int numPendingOutput = batchHolders.get(outBatchIndex).getNumPendingOutput();
-
+
if (!first && numPendingOutput == 0) {
this.outcome = IterOutcome.NONE;
return outcome;
@@ -445,17 +446,17 @@ public abstract class HashAggTemplate implements HashAggregator {
boolean outputKeysStatus = true;
boolean outputValuesStatus = true;
-
+
outputValuesStatus = batchHolders.get(outBatchIndex).outputValues(outStartIdxHolder, outNumRecordsHolder);
int numOutputRecords = outNumRecordsHolder.value;
-
+
if (EXTRA_DEBUG_1) {
logger.debug("After output values: outStartIdx = {}, outNumRecords = {}", outStartIdxHolder.value, outNumRecordsHolder.value);
}
if (outputValuesStatus) {
outputKeysStatus = this.htable.outputKeys(outBatchIndex, this.outContainer, outStartIdxHolder.value, outNumRecordsHolder.value) ;
}
-
+
if (outputKeysStatus && outputValuesStatus) {
// set the value count for outgoing batch value vectors
@@ -485,10 +486,10 @@ public abstract class HashAggTemplate implements HashAggregator {
}
} else {
if (!outputKeysStatus) {
- logger.debug("Failed to output keys for current batch index: {} ", outBatchIndex);
+ logger.debug("Failed to output keys for current batch index: {} ", outBatchIndex);
for(VectorWrapper<?> v : outContainer) {
logger.debug("At the time of failure, size of valuevector in outContainer = {}.", v.getValueVector().getValueCapacity());
- }
+ }
context.fail(new Exception("Failed to output keys for current batch !"));
}
if (!outputValuesStatus) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
index aff9751..d9499c7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -31,10 +31,10 @@ public interface HashTable {
public static TemplateClassDefinition<HashTable> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashTable>(HashTable.class, HashTableTemplate.class);
/** The initial default capacity of the hash table (in terms of number of buckets). */
- static final public int DEFAULT_INITIAL_CAPACITY = 1 << 16;
+ static final public int DEFAULT_INITIAL_CAPACITY = 1 << 16;
/** The maximum capacity of the hash table (in terms of number of buckets). */
- static final public int MAXIMUM_CAPACITY = 1 << 30;
+ static final public int MAXIMUM_CAPACITY = 1 << 30;
/** The default load factor of a hash table. */
static final public float DEFAULT_LOAD_FACTOR = 0.75f;
@@ -46,11 +46,11 @@ public interface HashTable {
static final public int BATCH_MASK = 0x0000FFFF;
public void setup(HashTableConfig htConfig, FragmentContext context, BufferAllocator allocator,
- RecordBatch incomingBuild, RecordBatch incomingProbe,
+ RecordBatch incomingBuild, RecordBatch incomingProbe,
RecordBatch outgoing, VectorContainer htContainerOrig);
- public PutStatus put(int incomingRowIdx, IntHolder htIdxHolder, int retryCount);
-
+ public PutStatus put(int incomingRowIdx, IndexPointer htIdxHolder, int retryCount);
+
public int containsKey(int incomingRowIdx, boolean isProbe);
public void getStats(HashTableStats stats);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
index e53ce05..e6c55bd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -43,7 +43,7 @@ public abstract class HashTableTemplate implements HashTable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashTable.class);
private static final boolean EXTRA_DEBUG = false;
-
+
private static final int EMPTY_SLOT = -1;
// private final int MISSING_VALUE = 65544;
@@ -68,7 +68,7 @@ public abstract class HashTableTemplate implements HashTable {
private int freeIndex = 0;
// Placeholder for the current index while probing the hash table
- private IntHolder currentIdxHolder;
+ private IndexPointer currentIdxHolder;
private FragmentContext context;
@@ -92,9 +92,9 @@ public abstract class HashTableTemplate implements HashTable {
private MaterializedField dummyIntField;
private int numResizing = 0;
-
+
private int resizingTime = 0;
-
+
// This class encapsulates the links, keys and values for up to BATCH_SIZE
// *unique* records. Thus, suppose there are N incoming record batches, each
// of size BATCH_SIZE..but they have M unique keys altogether, the number of
@@ -114,11 +114,11 @@ public abstract class HashTableTemplate implements HashTable {
private int batchOutputCount = 0;
private int batchIndex = 0;
-
+
private BatchHolder(int idx) {
this.batchIndex = idx;
-
+
if (idx == 0) { // first batch holder can use the original htContainer
htContainer = htContainerOrig;
} else { // otherwise create a new one using the original's fields
@@ -153,8 +153,8 @@ public abstract class HashTableTemplate implements HashTable {
// at the incomingRowIdx. if the key does not match, update the
// currentIdxHolder with the index of the next link.
private boolean isKeyMatch(int incomingRowIdx,
- IntHolder currentIdxHolder,
- boolean isProbe) {
+ IndexPointer currentIdxHolder,
+ boolean isProbe) {
int currentIdxWithinBatch = currentIdxHolder.value & BATCH_MASK;
boolean match = false;
@@ -164,7 +164,7 @@ public abstract class HashTableTemplate implements HashTable {
}
assert (currentIdxWithinBatch < HashTable.BATCH_SIZE);
assert (incomingRowIdx < HashTable.BATCH_SIZE);
-
+
if (isProbe)
match = isKeyMatchInternalProbe(incomingRowIdx, currentIdxWithinBatch);
else
@@ -213,14 +213,14 @@ public abstract class HashTableTemplate implements HashTable {
int size = links.getAccessor().getValueCount();
IntVector newLinks = allocMetadataVector(size, EMPTY_SLOT);
IntVector newHashValues = allocMetadataVector(size, 0);
-
+
for (int i = 0; i <= maxOccupiedIdx; i++) {
int entryIdxWithinBatch = i;
int entryIdx = entryIdxWithinBatch + batchStartIdx;
int hash = hashValues.getAccessor().get(entryIdxWithinBatch); // get the already saved hash value
int bucketIdx = getBucketIndex(hash, numbuckets);
int newStartIdx = newStartIndices.getAccessor().get(bucketIdx);
-
+
if (newStartIdx == EMPTY_SLOT) { // new bucket was empty
newStartIndices.getMutator().setSafe(bucketIdx, entryIdx); // update the start index to point to entry
newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT);
@@ -240,7 +240,7 @@ public abstract class HashTableTemplate implements HashTable {
int batchIdx = ((idx >>> 16) & BATCH_MASK);
bh = batchHolders.get(batchIdx);
}
-
+
if (bh == this && newLinks.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
newLinks.getMutator().setSafe(idxWithinBatch, entryIdx);
newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT);
@@ -253,7 +253,7 @@ public abstract class HashTableTemplate implements HashTable {
bh.links.getMutator().setSafe(idxWithinBatch, entryIdx); // update the link in the other batch
newLinks.getMutator().setSafe(entryIdxWithinBatch, EMPTY_SLOT); // update the newLink entry in this batch to mark end of the hash chain
newHashValues.getMutator().setSafe(entryIdxWithinBatch, hash);
-
+
if (EXTRA_DEBUG) logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
break;
@@ -282,19 +282,19 @@ public abstract class HashTableTemplate implements HashTable {
BigIntVector vv0 = getValueVector(0);
BigIntHolder holder = new BigIntHolder();
*/
-
+
// set the value count for htContainer's value vectors before the transfer ..
setValueCount();
-
+
Iterator<VectorWrapper<?>> outgoingIter = outContainer.iterator();
-
+
for (VectorWrapper<?> sourceWrapper : htContainer) {
ValueVector sourceVV = sourceWrapper.getValueVector();
ValueVector targetVV = outgoingIter.next().getValueVector();
TransferPair tp = sourceVV.makeTransferPair(targetVV);
tp.splitAndTransfer(outStartIndex, numRecords);
}
-
+
/*
logger.debug("Attempting to output keys for batch index: {} from index {} to maxOccupiedIndex {}.", this.batchIndex, 0, maxOccupiedIdx);
for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
@@ -312,17 +312,17 @@ public abstract class HashTableTemplate implements HashTable {
return false;
}
}
- */
+ */
return true;
}
private void setValueCount() {
for (VectorWrapper<?> vw : htContainer) {
ValueVector vv = vw.getValueVector();
- vv.getMutator().setValueCount(maxOccupiedIdx + 1);
+ vv.getMutator().setValueCount(maxOccupiedIdx + 1);
}
}
-
+
private void dump(int idx) {
while (true) {
int idxWithinBatch = idx & BATCH_MASK;
@@ -340,8 +340,8 @@ public abstract class HashTableTemplate implements HashTable {
links.clear();
hashValues.clear();
}
-
- // Only used for internal debugging. Get the value vector at a particular index from the htContainer.
+
+ // Only used for internal debugging. Get the value vector at a particular index from the htContainer.
// By default this assumes the VV is a BigIntVector.
private ValueVector getValueVector(int index) {
Object tmp = (htContainer).getValueAccessorById(BigIntVector.class, index).getValueVector();
@@ -413,23 +413,23 @@ public abstract class HashTableTemplate implements HashTable {
doSetup(incomingBuild, incomingProbe);
- currentIdxHolder = new IntHolder();
+ currentIdxHolder = new IndexPointer();
}
public int numBuckets() {
return startIndices.getAccessor().getValueCount();
}
- public int numResizing() {
- return numResizing;
+ public int numResizing() {
+ return numResizing;
}
public int size() {
return numEntries;
}
- public void getStats(HashTableStats stats) {
- assert stats != null;
+ public void getStats(HashTableStats stats) {
+ assert stats != null;
stats.numBuckets = numBuckets();
stats.numEntries = numEntries;
stats.numResizing = numResizing;
@@ -466,8 +466,8 @@ public abstract class HashTableTemplate implements HashTable {
return rounded;
}
-
- public PutStatus put(int incomingRowIdx, IntHolder htIdxHolder, int retryCount) {
+
+ public PutStatus put(int incomingRowIdx, IndexPointer htIdxHolder, int retryCount) {
HashTable.PutStatus putStatus = put(incomingRowIdx, htIdxHolder) ;
int count = retryCount;
int numBatchHolders;
@@ -482,7 +482,7 @@ public abstract class HashTableTemplate implements HashTable {
return putStatus;
}
- private PutStatus put(int incomingRowIdx, IntHolder htIdxHolder) {
+ private PutStatus put(int incomingRowIdx, IndexPointer htIdxHolder) {
int hash = getHashBuild(incomingRowIdx);
hash = Math.abs(hash);
@@ -587,7 +587,7 @@ public abstract class HashTableTemplate implements HashTable {
int i = getBucketIndex(hash, numBuckets());
int currentIdx = startIndices.getAccessor().get(i);
-
+
if (currentIdx == EMPTY_SLOT) {
return -1;
}
@@ -604,7 +604,7 @@ public abstract class HashTableTemplate implements HashTable {
} else if (currentIdxHolder.value == EMPTY_SLOT) {
break;
} else {
- bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
+ bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
}
}
@@ -641,7 +641,7 @@ public abstract class HashTableTemplate implements HashTable {
private void resizeAndRehashIfNeeded() {
if (numEntries < threshold)
return;
-
+
long t0 = System.currentTimeMillis();
if (EXTRA_DEBUG) logger.debug("Hash table numEntries = {}, threshold = {}; resizing the table...", numEntries, threshold);
@@ -695,7 +695,7 @@ public abstract class HashTableTemplate implements HashTable {
}
return true;
}
-
+
private IntVector allocMetadataVector(int size, int initialValue) {
IntVector vector = (IntVector) TypeHelper.getNewVector(dummyIntField, allocator);
vector.allocateNew(size);
[20/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderReplacementVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderReplacementVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderReplacementVisitor.java
new file mode 100644
index 0000000..2cec537
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderReplacementVisitor.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import java.io.PrintWriter;
+
+import org.apache.commons.io.output.StringBuilderWriter;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.util.Textifier;
+import org.objectweb.asm.util.TraceMethodVisitor;
+
+public class ValueHolderReplacementVisitor extends ClassVisitor {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderReplacementVisitor.class);
+
+ public ValueHolderReplacementVisitor(ClassVisitor cw) {
+ super(Opcodes.ASM4, cw);
+ }
+
+
+
+ @Override
+ public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+ MethodVisitor innerVisitor = super.visitMethod(access, name, desc, signature, exceptions);
+// innerVisitor = new Debugger(access, name, desc, signature, exceptions, innerVisitor);
+ return new ScalarReplacementNode(access, name, desc, signature, exceptions, innerVisitor);
+ }
+
+ private static class Debugger extends MethodNode {
+
+ MethodVisitor inner;
+
+ public Debugger(int access, String name, String desc, String signature, String[] exceptions, MethodVisitor inner) {
+ super(access, name, desc, signature, exceptions);
+ this.inner = inner;
+
+ }
+
+ @Override
+ public void visitEnd() {
+ try{
+ accept(inner);
+ }catch(Exception e){
+ Textifier t = new Textifier();
+ accept(new TraceMethodVisitor(t));
+ StringBuilderWriter sw = new StringBuilderWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ t.print(pw);
+ pw.flush();
+ String bytecode = sw.getBuilder().toString();
+ logger.error(String.format("Failure while rendering method %s, %s, %s. ByteCode:\n %s", name, desc, signature, bytecode), e);
+ throw new RuntimeException(String.format("Failure while rendering method %s, %s, %s. ByteCode:\n %s", name, desc, signature, bytecode), e);
+ }
+
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java
index 9fe0cf3..a7b5680 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java
@@ -28,13 +28,21 @@ import com.thoughtworks.paranamer.Paranamer;
public class CodeGeneratorMethod implements Iterable<CodeGeneratorArgument>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGeneratorMethod.class);
-
+
private final String methodName;
private final Class<?> returnType;
private final CodeGeneratorArgument[] arguments;
private final Class<?>[] exs;
private final Method underlyingMethod;
-
+
+ public CodeGeneratorMethod(String name, Class<?> returnType){
+ this.methodName = name;
+ this.returnType = returnType;
+ this.exs = new Class<?>[0];
+ this.underlyingMethod = null;
+ this.arguments = new CodeGeneratorArgument[0];
+ }
+
public CodeGeneratorMethod(Method m){
this.underlyingMethod = m;
this.methodName = m.getName();
@@ -51,7 +59,7 @@ public class CodeGeneratorMethod implements Iterable<CodeGeneratorArgument>{
}
exs = m.getExceptionTypes();
}
-
+
public String getMethodName() {
return methodName;
}
@@ -62,7 +70,7 @@ public class CodeGeneratorMethod implements Iterable<CodeGeneratorArgument>{
public Iterable<Class<?>> getThrowsIterable(){
return ImmutableList.copyOf(exs);
}
-
+
@Override
public Iterator<CodeGeneratorArgument> iterator() {
return Iterators.forArray(arguments);
@@ -73,5 +81,5 @@ public class CodeGeneratorMethod implements Iterable<CodeGeneratorArgument>{
return "CodeGeneratorMethod [" + underlyingMethod.toGenericString() + "]";
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
index 70a3cda..57551ed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
@@ -28,7 +28,7 @@ public class GeneratorMapping {
private String eval;
private String reset;
private String cleanup;
-
+
public GeneratorMapping(String setup, String eval, String reset, String cleanup) {
super();
@@ -49,15 +49,15 @@ public class GeneratorMapping {
public static GeneratorMapping GM(String setup, String eval){
return create(setup, eval, null, null);
}
-
+
public static GeneratorMapping GM(String setup, String eval, String reset, String cleanup){
return create(setup, eval, reset, cleanup);
}
- public static GeneratorMapping create(String setup, String eval, String reset, String cleanup){
+ public static GeneratorMapping create(String setup, String eval, String reset, String cleanup){
return new GeneratorMapping(setup, eval, reset, cleanup);
}
-
+
public String getMethodName(BlockType type){
switch(type){
case CLEANUP:
@@ -76,6 +76,6 @@ public class GeneratorMapping {
throw new IllegalStateException();
}
}
-
-
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
index ac0e413..80aaae8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
@@ -42,6 +42,10 @@ public class MappingSet {
this("inIndex", "outIndex", new GeneratorMapping[] { mapping, mapping });
}
+ public boolean hasEmbeddedConstant(){
+ return constant == current;
+ }
+
public MappingSet(String readIndex, String writeIndex, String workspaceIndex, String incoming, String outgoing,
String workspace, GeneratorMapping... mappings) {
this(readIndex, writeIndex, incoming, outgoing, mappings);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
index 536370b..d397acc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
@@ -31,13 +31,15 @@ import com.google.common.collect.Maps;
public class SignatureHolder implements Iterable<CodeGeneratorMethod>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SignatureHolder.class);
-
+
private final Class<?> signature;
private final CodeGeneratorMethod[] methods;
private final Map<String, Integer> methodMap;
private final SignatureHolder[] childHolders;
-
-
+
+ public static final String DRILL_INIT_METHOD = "__DRILL_INIT__";
+ public static final CodeGeneratorMethod DRILL_INIT = new CodeGeneratorMethod(DRILL_INIT_METHOD, void.class);
+
public static SignatureHolder getHolder(Class<?> signature){
List<SignatureHolder> innerClasses = Lists.newArrayList();
for(Class<?> inner : signature.getClasses()){
@@ -46,36 +48,39 @@ public class SignatureHolder implements Iterable<CodeGeneratorMethod>{
}
return new SignatureHolder(signature, innerClasses.toArray(new SignatureHolder[innerClasses.size()]));
}
-
+
private SignatureHolder(Class<?> signature, SignatureHolder[] childHolders){
this.childHolders = childHolders;
this.signature = signature;
- Map<String, Integer> newMap = Maps.newHashMap();
-
+ Map<String, Integer> newMap = Maps.newHashMap();
+
List<CodeGeneratorMethod> methodHolders = Lists.newArrayList();
Method[] reflectMethods = signature.getDeclaredMethods();
-
+
for(Method m : reflectMethods){
if( (m.getModifiers() & Modifier.ABSTRACT) == 0 && m.getAnnotation(RuntimeOverridden.class) == null) continue;
methodHolders.add(new CodeGeneratorMethod(m));
}
-
- methods = new CodeGeneratorMethod[methodHolders.size()];
+
+ methods = new CodeGeneratorMethod[methodHolders.size()+1];
for(int i =0; i < methodHolders.size(); i++){
methods[i] = methodHolders.get(i);
Integer old = newMap.put(methods[i].getMethodName(), i);
if(old != null) throw new IllegalStateException(String.format("Attempting to add a method with name %s when there is already one method of that name in this class that is set to be runtime generated.", methods[i].getMethodName()));
-
+
}
+ methods[methodHolders.size()] = DRILL_INIT;
+ newMap.put(DRILL_INIT.getMethodName(), methodHolders.size());
+
methodMap = ImmutableMap.copyOf(newMap);
}
-
+
public Class<?> getSignatureClass(){
return signature;
}
-
+
public CodeGeneratorMethod get(int i){
return methods[i];
}
@@ -84,12 +89,12 @@ public class SignatureHolder implements Iterable<CodeGeneratorMethod>{
public Iterator<CodeGeneratorMethod> iterator() {
return Iterators.forArray(methods);
}
-
+
public int size(){
return methods.length;
}
-
-
+
+
public SignatureHolder[] getChildHolders() {
return childHolders;
}
@@ -109,6 +114,6 @@ public class SignatureHolder implements Iterable<CodeGeneratorMethod>{
return "SignatureHolder [methods="
+ (methods != null ? Arrays.asList(methods).subList(0, Math.min(methods.length, maxLen)) : null) + "]";
}
-
-
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
index 4d4fc99..69d68bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
@@ -275,7 +275,7 @@ public class ClassGenerator<T>{
for(ClassGenerator<T> child : innerClasses.values()) {
child.flushCode();
- }
+ }
}
public JCodeModel getModel() {
@@ -405,6 +405,10 @@ public class ClassGenerator<T>{
return this;
}
+ public JFieldRef f(String name){
+ return holder.ref(name);
+ }
+
public boolean isConstant() {
return this.isConstant;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
index 658c63c..73c5980 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
@@ -32,10 +32,10 @@ import com.sun.codemodel.JDefinedClass;
* A code generator is responsible for generating the Java source code required to complete the implementation of an
* abstract template. It is used with a class transformer to merge precompiled template code with runtime generated and
* compiled query specific code to create a runtime instance.
- *
+ *
* A code generator can contain one or more ClassGenerators that implement outer and inner classes associated with a
* particular runtime generated instance.
- *
+ *
* @param <T>
* The interface that results from compiling and merging the runtime code that is generated.
*/
@@ -46,8 +46,11 @@ public class CodeGenerator<T> {
private final TemplateClassDefinition<T> definition;
private final String className;
private final String fqcn;
+
private final JCodeModel model;
private final ClassGenerator<T> rootGenerator;
+ private String generatedCode;
+ private String generifiedCode;
CodeGenerator(TemplateClassDefinition<T> definition, FunctionImplementationRegistry funcRegistry) {
this(ClassGenerator.getDefaultMapping(), definition, funcRegistry);
@@ -74,12 +77,24 @@ public class CodeGenerator<T> {
return rootGenerator;
}
- public String generate() throws IOException {
+ public void generate() throws IOException {
rootGenerator.flushCode();
SingleClassStringWriter w = new SingleClassStringWriter();
model.build(w);
- return w.getCode().toString();
+
+ this.generatedCode = w.getCode().toString();
+ this.generifiedCode = generatedCode.replaceAll(this.className, "GenericGenerated");
+
+ }
+
+ public String generateAndGet() throws IOException {
+ generate();
+ return generatedCode;
+ }
+
+ public String getGeneratedCode(){
+ return generatedCode;
}
public TemplateClassDefinition<T> getDefinition() {
@@ -99,15 +114,49 @@ public class CodeGenerator<T> {
FunctionImplementationRegistry funcRegistry){
return get(definition, funcRegistry).getRoot();
}
-
+
public static <T> ClassGenerator<T> getRoot(MappingSet mappingSet, TemplateClassDefinition<T> definition,
FunctionImplementationRegistry funcRegistry){
return get(mappingSet, definition, funcRegistry).getRoot();
}
-
+
public static <T> CodeGenerator<T> get(MappingSet mappingSet, TemplateClassDefinition<T> definition,
FunctionImplementationRegistry funcRegistry) {
return new CodeGenerator<T>(mappingSet, definition, funcRegistry);
}
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((definition == null) ? 0 : definition.hashCode());
+ result = prime * result + ((generatedCode == null) ? 0 : generatedCode.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ CodeGenerator other = (CodeGenerator) obj;
+ if (definition == null) {
+ if (other.definition != null)
+ return false;
+ } else if (!definition.equals(other.definition))
+ return false;
+ if (generatedCode == null) {
+ if (other.generatedCode != null)
+ return false;
+ } else if (!generatedCode.equals(other.generatedCode))
+ return false;
+ return true;
+ }
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index caf9179..cf39246 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.expr;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -80,7 +81,13 @@ public class EvaluationVisitor {
}
public HoldingContainer addExpr(LogicalExpression e, ClassGenerator<?> generator) {
- Set<LogicalExpression> constantBoundaries = ConstantExpressionIdentifier.getConstantExpressionSet(e);
+
+ Set<LogicalExpression> constantBoundaries;
+ if(generator.getMappingSet().hasEmbeddedConstant()){
+ constantBoundaries = Collections.emptySet();
+ }else{
+ constantBoundaries = ConstantExpressionIdentifier.getConstantExpressionSet(e);
+ }
return e.accept(new ConstantFilter(constantBoundaries), generator);
}
@@ -93,17 +100,17 @@ public class EvaluationVisitor {
}
@Override
- public HoldingContainer visitBooleanOperator(BooleanOperator op,
+ public HoldingContainer visitBooleanOperator(BooleanOperator op,
ClassGenerator<?> generator) throws RuntimeException {
if (op.getName().equals("booleanAnd")) {
return visitBooleanAnd(op, generator);
}else if(op.getName().equals("booleanOr")) {
- return visitBooleanOr(op, generator);
+ return visitBooleanOr(op, generator);
} else {
throw new UnsupportedOperationException("BooleanOperator can only be booleanAnd, booleanOr. You are using " + op.getName());
}
}
-
+
@Override
public HoldingContainer visitFunctionHolderExpression(FunctionHolderExpression holderExpr,
ClassGenerator<?> generator) throws RuntimeException {
@@ -290,16 +297,9 @@ public class EvaluationVisitor {
return outputContainer;
}
} else {
- String setMethod = e.isSafe() ? "setSafe" : "set";
- String isSafeMethod = "isSafe";
-
- JInvocation setMeth;
- if (Types.usesHolderForGet(inputContainer.getMajorType())) {
- setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getHolder());
- } else {
- setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getValue());
- }
+ final JInvocation setMeth = GetSetVectorHelper.write(e.getChild().getMajorType(), vv, inputContainer, outIndex, e.isSafe() ? "setSafe" : "set");
+ final String isSafeMethod = "isSafe";
if (e.isSafe()) {
HoldingContainer outputContainer = generator.declare(Types.REQUIRED_BIT);
@@ -356,24 +356,9 @@ public class EvaluationVisitor {
}
if (!hasReadPath && !complex) {
- JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get");
- JInvocation getValueAccessor2 = vv1.invoke("getAccessor");
JBlock eval = new JBlock();
-
- if (primitive) {
- eval.assign(out.getValue(), getValueAccessor.arg(indexVariable));
- } else {
- eval.add(getValueAccessor.arg(indexVariable).arg(out.getHolder()));
- }
-
- if (out.isOptional()) {
- JBlock blk = generator.getEvalBlock();
- blk.assign(out.getIsSet(), getValueAccessor2.invoke("isSet").arg(indexVariable));
- JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1)));
- jc._then().add(eval);
- } else {
- generator.getEvalBlock().add(eval);
- }
+ GetSetVectorHelper.read(e.getMajorType(), vv1, eval, out, generator.getModel(), indexVariable);
+ generator.getEvalBlock().add(eval);
} else {
JExpression vector = e.isSuperReader() ? vv1.component(componentVariable) : vv1;
@@ -419,7 +404,7 @@ public class EvaluationVisitor {
JBlock ifNoVal = eval._if(desiredIndex.ne(currentIndex))._then().block();
if (out.isOptional()) {
ifNoVal.assign(out.getIsSet(), JExpr.lit(0));
- }
+ }
ifNoVal.assign(isNull, JExpr.lit(1));
ifNoVal._break(label);
@@ -503,8 +488,9 @@ public class EvaluationVisitor {
JType holderType = generator.getHolderType(majorType);
JVar var = generator.declareClassField("string", holderType);
JExpression stringLiteral = JExpr.lit(e.value);
+ JExpression buffer = generator.getMappingSet().getIncoming().invoke("getContext").invoke("getManagedBuffer");
setup.assign(var,
- generator.getModel().ref(ValueHolderHelper.class).staticInvoke("getVarCharHolder").arg(stringLiteral));
+ generator.getModel().ref(ValueHolderHelper.class).staticInvoke("getVarCharHolder").arg(buffer).arg(stringLiteral));
return new HoldingContainer(majorType, var, null, null);
}
@@ -601,20 +587,20 @@ public class EvaluationVisitor {
FunctionCall fc = new FunctionCall(convertFunctionName, newArgs, e.getPosition());
return fc.accept(this, value);
}
-
+
private HoldingContainer visitBooleanAnd(BooleanOperator op,
ClassGenerator<?> generator) {
-
+
HoldingContainer out = generator.declare(op.getMajorType());
-
+
JLabel label = generator.getEvalBlockLabel("AndOP");
JBlock eval = generator.getEvalBlock().block(); // enter into nested block
generator.nestEvalBlock(eval);
HoldingContainer arg = null;
-
+
JExpression e = null;
-
+
// value of boolean "and" when one side is null
// p q p and q
// true null null
@@ -624,57 +610,57 @@ public class EvaluationVisitor {
// null null null
for (int i = 0; i < op.args.size(); i++) {
arg = op.args.get(i).accept(this, generator);
-
+
JBlock earlyExit = null;
if (arg.isOptional()) {
- earlyExit = eval._if(arg.getIsSet().eq(JExpr.lit(1)).cand(arg.getValue().ne(JExpr.lit(1))))._then();
+ earlyExit = eval._if(arg.getIsSet().eq(JExpr.lit(1)).cand(arg.getValue().ne(JExpr.lit(1))))._then();
if(e == null){
e = arg.getIsSet();
}else{
e = e.mul(arg.getIsSet());
- }
+ }
} else {
- earlyExit = eval._if(arg.getValue().ne(JExpr.lit(1)))._then();
+ earlyExit = eval._if(arg.getValue().ne(JExpr.lit(1)))._then();
}
-
+
if (out.isOptional()) {
earlyExit.assign(out.getIsSet(), JExpr.lit(1));
- }
-
- earlyExit.assign(out.getValue(), JExpr.lit(0));
- earlyExit._break(label);
+ }
+
+ earlyExit.assign(out.getValue(), JExpr.lit(0));
+ earlyExit._break(label);
}
-
+
if (out.isOptional()) {
assert (e != null);
-
+
JConditional notSetJC = eval._if(e.eq(JExpr.lit(0)));
notSetJC._then().assign(out.getIsSet(), JExpr.lit(0));
-
+
JBlock setBlock = notSetJC._else().block();
setBlock.assign(out.getIsSet(), JExpr.lit(1));
setBlock.assign(out.getValue(), JExpr.lit(1));
} else {
- assert (e == null);
+ assert (e == null);
eval.assign(out.getValue(), JExpr.lit(1)) ;
}
-
- generator.unNestEvalBlock(); // exit from nested block
-
+
+ generator.unNestEvalBlock(); // exit from nested block
+
return out;
}
-
+
private HoldingContainer visitBooleanOr(BooleanOperator op,
ClassGenerator<?> generator) {
-
+
HoldingContainer out = generator.declare(op.getMajorType());
-
+
JLabel label = generator.getEvalBlockLabel("OrOP");
JBlock eval = generator.getEvalBlock().block();
- generator.nestEvalBlock(eval); // enter into nested block.
+ generator.nestEvalBlock(eval); // enter into nested block.
HoldingContainer arg = null;
-
+
JExpression e = null;
// value of boolean "or" when one side is null
@@ -684,49 +670,49 @@ public class EvaluationVisitor {
// null true true
// null false null
// null null null
-
+
for (int i = 0; i < op.args.size(); i++) {
arg = op.args.get(i).accept(this, generator);
-
+
JBlock earlyExit = null;
if (arg.isOptional()) {
- earlyExit = eval._if(arg.getIsSet().eq(JExpr.lit(1)).cand(arg.getValue().eq(JExpr.lit(1))))._then();
+ earlyExit = eval._if(arg.getIsSet().eq(JExpr.lit(1)).cand(arg.getValue().eq(JExpr.lit(1))))._then();
if(e == null){
e = arg.getIsSet();
}else{
e = e.mul(arg.getIsSet());
- }
+ }
} else {
- earlyExit = eval._if(arg.getValue().eq(JExpr.lit(1)))._then();
+ earlyExit = eval._if(arg.getValue().eq(JExpr.lit(1)))._then();
}
-
+
if (out.isOptional()) {
earlyExit.assign(out.getIsSet(), JExpr.lit(1));
- }
-
- earlyExit.assign(out.getValue(), JExpr.lit(1));
- earlyExit._break(label);
+ }
+
+ earlyExit.assign(out.getValue(), JExpr.lit(1));
+ earlyExit._break(label);
}
-
+
if (out.isOptional()) {
assert (e != null);
-
+
JConditional notSetJC = eval._if(e.eq(JExpr.lit(0)));
notSetJC._then().assign(out.getIsSet(), JExpr.lit(0));
-
+
JBlock setBlock = notSetJC._else().block();
setBlock.assign(out.getIsSet(), JExpr.lit(1));
setBlock.assign(out.getValue(), JExpr.lit(0));
} else {
- assert (e == null);
+ assert (e == null);
eval.assign(out.getValue(), JExpr.lit(0)) ;
}
-
- generator.unNestEvalBlock(); // exit from nested block.
-
+
+ generator.unNestEvalBlock(); // exit from nested block.
+
return out;
}
-
+
}
private class ConstantFilter extends EvalVisitor {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/GetSetVectorHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/GetSetVectorHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/GetSetVectorHelper.java
new file mode 100644
index 0000000..46bd7f0
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/GetSetVectorHelper.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr;
+
+import io.netty.buffer.DrillBuf;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+
+import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JCodeModel;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JVar;
+
+public class GetSetVectorHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GetSetVectorHelper.class);
+
+ // eval.add(getValueAccessor.arg(indexVariable).arg(out.getHolder()));
+
+ public static void read(MajorType type, JExpression vector, JBlock eval, HoldingContainer out, JCodeModel model,
+ JExpression indexVariable) {
+
+ JInvocation getValueAccessor = vector.invoke("getAccessor");
+
+ switch(type.getMode()){
+ case OPTIONAL:
+ eval.assign(out.getIsSet(), getValueAccessor.invoke("isSet").arg(indexVariable));
+ eval = eval._if(out.getIsSet().eq(JExpr.lit(1)))._then();
+
+ // fall through
+ case REQUIRED:
+ switch (type.getMinorType()) {
+ case BIGINT:
+ case FLOAT4:
+ case FLOAT8:
+ case INT:
+ case MONEY:
+ case SMALLINT:
+ case TINYINT:
+ case UINT1:
+ case UINT2:
+ case UINT4:
+ case UINT8:
+ case INTERVALYEAR:
+ case DATE:
+ case TIME:
+ case TIMESTAMP:
+ case BIT:
+ eval.assign(out.getValue(), getValueAccessor.invoke("get").arg(indexVariable));
+ return;
+ case DECIMAL9:
+ case DECIMAL18:
+ eval.assign(out.getHolder().ref("scale"), vector.invoke("getField").invoke("getScale"));
+ eval.assign(out.getHolder().ref("precision"), vector.invoke("getField").invoke("getPrecision"));
+ eval.assign(out.getValue(), getValueAccessor.invoke("get").arg(indexVariable));
+ return;
+ case DECIMAL28DENSE:
+ case DECIMAL28SPARSE:
+ case DECIMAL38DENSE:
+ case DECIMAL38SPARSE:
+ eval.assign(out.getHolder().ref("scale"), vector.invoke("getField").invoke("getScale"));
+ eval.assign(out.getHolder().ref("precision"), vector.invoke("getField").invoke("getPrecision"));
+ eval.assign(out.getHolder().ref("start"), JExpr.lit(TypeHelper.getSize(type)).mul(indexVariable));
+ eval.assign(out.getHolder().ref("buffer"), vector.invoke("getData"));
+ return;
+ case INTERVAL:{
+ JVar start = eval.decl(model.INT, "start", JExpr.lit(TypeHelper.getSize(type)).mul(indexVariable));
+ JVar data = eval.decl(model.ref(DrillBuf.class), "data", vector.invoke("getData"));
+ eval.assign(out.getHolder().ref("months"), data.invoke("getInt").arg(start));
+ eval.assign(out.getHolder().ref("days"), data.invoke("getInt").arg(start.plus(JExpr.lit(4))));
+ eval.assign(out.getHolder().ref("milliseconds"), data.invoke("getInt").arg(start.plus(JExpr.lit(8))));
+ return;
+ }
+ case INTERVALDAY: {
+ JVar start = eval.decl(model.INT, "start", JExpr.lit(TypeHelper.getSize(type)).mul(indexVariable));
+ eval.assign(out.getHolder().ref("days"), vector.invoke("getData").invoke("getInt").arg(start));
+ eval.assign(out.getHolder().ref("milliseconds"), vector.invoke("getData").invoke("getInt").arg(start.plus(JExpr.lit(4))));
+ return;
+ }
+ case VAR16CHAR:
+ case VARBINARY:
+ case VARCHAR:
+ eval.assign(out.getHolder().ref("buffer"), vector.invoke("getData"));
+ JVar se = eval.decl(model.LONG, "startEnd", getValueAccessor.invoke("getStartEnd").arg(indexVariable));
+ eval.assign(out.getHolder().ref("start"), JExpr.cast(model._ref(int.class), se));
+ eval.assign(out.getHolder().ref("end"), JExpr.cast(model._ref(int.class), se.shr(JExpr.lit(32))));
+ return;
+
+ }
+ }
+
+ // fallback.
+ eval.add(getValueAccessor.invoke("get").arg(indexVariable).arg(out.getHolder()));
+ }
+
+ public static JInvocation write(MajorType type, JVar vector, HoldingContainer in, JExpression indexVariable, String setMethodName) {
+
+ JInvocation setMethod = vector.invoke("getMutator").invoke(setMethodName).arg(indexVariable);
+
+ switch(type.getMode()){
+ case OPTIONAL:
+ setMethod = setMethod.arg(in.f("isSet"));
+ case REQUIRED:
+ switch (type.getMinorType()) {
+ case BIGINT:
+ case FLOAT4:
+ case FLOAT8:
+ case INT:
+ case MONEY:
+ case SMALLINT:
+ case TINYINT:
+ case UINT1:
+ case UINT2:
+ case UINT4:
+ case UINT8:
+ case INTERVALYEAR:
+ case DATE:
+ case TIME:
+ case TIMESTAMP:
+ case BIT:
+ case DECIMAL9:
+ case DECIMAL18:
+ return setMethod //
+ .arg(in.getValue());
+ case DECIMAL28DENSE:
+ case DECIMAL28SPARSE:
+ case DECIMAL38DENSE:
+ case DECIMAL38SPARSE:
+ return setMethod //
+ .arg(in.f("start")) //
+ .arg(in.f("buffer"));
+ case INTERVAL:{
+ return setMethod //
+ .arg(in.f("months")) //
+ .arg(in.f("days")) //
+ .arg(in.f("milliseconds"));
+ }
+ case INTERVALDAY: {
+ return setMethod //
+ .arg(in.f("days")) //
+ .arg(in.f("milliseconds"));
+ }
+ case VAR16CHAR:
+ case VARBINARY:
+ case VARCHAR:
+ return setMethod //
+ .arg(in.f("start")) //
+ .arg(in.f("end")) //
+ .arg(in.f("buffer"));
+ }
+ }
+
+
+ return setMethod.arg(in.getHolder());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
index 989e490..5281c37 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
@@ -45,7 +45,7 @@ import com.sun.codemodel.JVar;
class DrillAggFuncHolder extends DrillFuncHolder{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillAggFuncHolder.class);
-
+
private final String setup;
private final String reset;
private final String add;
@@ -57,7 +57,7 @@ class DrillAggFuncHolder extends DrillFuncHolder{
Map<String, String> methods, List<String> imports) {
this(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, FunctionCostCategory.getDefault());
}
-
+
public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
Map<String, String> methods, List<String> imports, FunctionCostCategory costCategory) {
@@ -84,38 +84,38 @@ class DrillAggFuncHolder extends DrillFuncHolder{
@Override
public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables) {
- if (!g.getMappingSet().isHashAggMapping()) { //Declare workspace vars for non-hash-aggregation.
+ if (!g.getMappingSet().isHashAggMapping()) { //Declare workspace vars for non-hash-aggregation.
JVar[] workspaceJVars = declareWorkspaceVariables(g);
- generateBody(g, BlockType.SETUP, setup, null, workspaceJVars, false);
+ generateBody(g, BlockType.SETUP, setup, null, workspaceJVars, true);
return workspaceJVars;
} else { //Declare workspace vars and workspace vectors for hash aggregation.
-
+
JVar[] workspaceJVars = declareWorkspaceVectors(g);
-
+
JBlock setupBlock = g.getSetupBlock();
-
- //Loop through all workspace vectors, to get the minimum of size of all workspace vectors.
+
+ //Loop through all workspace vectors, to get the minimum of size of all workspace vectors.
JVar sizeVar = setupBlock.decl(g.getModel().INT, "vectorSize", JExpr.lit(Integer.MAX_VALUE));
JClass mathClass = g.getModel().ref(Math.class);
for (int id = 0; id<workspaceVars.length; id ++) {
setupBlock.assign(sizeVar,mathClass.staticInvoke("min").arg(sizeVar).arg(g.getWorkspaceVectors().get(workspaceVars[id]).invoke("getValueCapacity")));
}
-
- for(int i =0 ; i < workspaceVars.length; i++) {
+
+ for(int i =0 ; i < workspaceVars.length; i++) {
setupBlock.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
}
-
- //Use for loop to initialize entries in the workspace vectors.
+
+ //Use for loop to initialize entries in the workspace vectors.
JForLoop forLoop = setupBlock._for();
JVar ivar = forLoop.init(g.getModel().INT, "drill_internal_i", JExpr.lit(0));
forLoop.test(ivar.lt(sizeVar));
forLoop.update(ivar.assignPlus(JExpr.lit(1)));
-
+
JBlock subBlock = generateInitWorkspaceBlockHA(g, BlockType.SETUP, setup, workspaceJVars, ivar);
forLoop.body().add(subBlock);
- return workspaceJVars;
+ return workspaceJVars;
}
-
+
}
@@ -134,7 +134,7 @@ class DrillAggFuncHolder extends DrillFuncHolder{
JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type)));
addProtectedBlock(g, sub, output, null, workspaceJVars, false);
sub.assign(out.getHolder(), internalOutput);
- //hash aggregate uses workspace vectors. Initialization is done in "setup" and does not require "reset" block.
+ //hash aggregate uses workspace vectors. Initialization is done in "setup" and does not require "reset" block.
if (!g.getMappingSet().isHashAggMapping()) {
generateBody(g, BlockType.RESET, reset, null, workspaceJVars, false);
}
@@ -146,27 +146,27 @@ class DrillAggFuncHolder extends DrillFuncHolder{
private JVar[] declareWorkspaceVectors(ClassGenerator<?> g) {
JVar[] workspaceJVars = new JVar[workspaceVars.length];
-
+
for(int i =0 ; i < workspaceVars.length; i++){
Preconditions.checkState(Types.isFixedWidthType(workspaceVars[i].majorType), String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have variable length type.", workspaceVars[i].name, registeredNames[0]));
Preconditions.checkState(workspaceVars[i].majorType.getMode()==DataMode.REQUIRED, String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have null or repeated type.", workspaceVars[i].name, registeredNames[0]));
//workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType), JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
- workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType));
-
+ workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType));
+
//Declare a workspace vector for the workspace var.
TypedFieldId typedFieldId = new TypedFieldId(workspaceVars[i].majorType, g.getWorkspaceTypes().size());
JVar vv = g.declareVectorValueSetupAndMember(g.getMappingSet().getWorkspace(), typedFieldId);
-
+
g.getWorkspaceTypes().add(typedFieldId);
g.getWorkspaceVectors().put(workspaceVars[i], vv);
}
return workspaceJVars;
}
-
+
private JBlock generateInitWorkspaceBlockHA(ClassGenerator<?> g, BlockType bt, String body, JVar[] workspaceJVars, JExpression wsIndexVariable){
JBlock initBlock = new JBlock(true, true);
- if(!Strings.isNullOrEmpty(body) && !body.trim().isEmpty()){
+ if(!Strings.isNullOrEmpty(body) && !body.trim().isEmpty()){
JBlock sub = new JBlock(true, true);
addProtectedBlockHA(g, sub, body, null, workspaceJVars, wsIndexVariable);
initBlock.directStatement(String.format("/** start %s for function %s **/ ", bt.name(), registeredNames[0]));
@@ -175,51 +175,51 @@ class DrillAggFuncHolder extends DrillFuncHolder{
}
return initBlock;
}
-
+
@Override
protected void addProtectedBlock(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, boolean decConstantInputOnly){
if (!g.getMappingSet().isHashAggMapping()) {
super.addProtectedBlock(g, sub, body, inputVariables, workspaceJVars, decConstantInputOnly);
} else {
JExpression indexVariable = g.getMappingSet().getWorkspaceIndex();
- addProtectedBlockHA(g, sub, body, inputVariables, workspaceJVars, indexVariable);
+ addProtectedBlockHA(g, sub, body, inputVariables, workspaceJVars, indexVariable);
}
}
/*
- * This is customized version of "addProtectedBlock" for hash aggregation. It take one additional parameter "wsIndexVariable".
+ * This is customized version of "addProtectedBlock" for hash aggregation. It take one additional parameter "wsIndexVariable".
*/
private void addProtectedBlockHA(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, JExpression wsIndexVariable){
if (inputVariables != null){
for(int i =0; i < inputVariables.length; i++){
ValueReference parameter = parameters[i];
HoldingContainer inputVariable = inputVariables[i];
- sub.decl(inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
+ sub.decl(inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
}
}
- JVar[] internalVars = new JVar[workspaceJVars.length];
+ JVar[] internalVars = new JVar[workspaceJVars.length];
for(int i =0; i < workspaceJVars.length; i++){
//sub.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
- //Access workspaceVar through workspace vector.
- JInvocation getValueAccessor = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getAccessor").invoke("get");
+ //Access workspaceVar through workspace vector.
+ JInvocation getValueAccessor = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getAccessor").invoke("get");
if (Types.usesHolderForGet(workspaceVars[i].majorType)) {
sub.add(getValueAccessor.arg(wsIndexVariable).arg(workspaceJVars[i]));
} else {
sub.assign(workspaceJVars[i].ref("value"), getValueAccessor.arg(wsIndexVariable));
}
- internalVars[i] = sub.decl(g.getHolderType(workspaceVars[i].majorType), workspaceVars[i].name, workspaceJVars[i]);
+ internalVars[i] = sub.decl(g.getHolderType(workspaceVars[i].majorType), workspaceVars[i].name, workspaceJVars[i]);
}
-
+
Preconditions.checkNotNull(body);
sub.directStatement(body);
JVar successVar = sub.decl(JType.parse(g.getModel(), "boolean"), "success", JExpr.lit(false));
-
+
// reassign workspace variables back.
for(int i =0; i < workspaceJVars.length; i++){
sub.assign(workspaceJVars[i], internalVars[i]);
-
- //Change workspaceVar through workspace vector.
+
+ //Change workspaceVar through workspace vector.
JInvocation setMeth;
if (Types.usesHolderForGet(workspaceVars[i].majorType)) {
setMeth = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getMutator").invoke("setSafe").arg(wsIndexVariable).arg(workspaceJVars[i]);
@@ -233,7 +233,7 @@ class DrillAggFuncHolder extends DrillFuncHolder{
sub._if(successVar.eq(JExpr.lit(false)))._then()._throw(JExpr._new(drillRunTimeException).arg(JExpr.lit("setsafe() failed; cannot set holder value into the vector")));
}
-
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java
index 98c1c85..61e04ca 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java
@@ -23,12 +23,11 @@ import java.util.Map;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
-
import org.apache.drill.common.util.DecimalScalePrecisionAddFunction;
import org.apache.drill.common.util.DecimalScalePrecisionDivideFunction;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.util.DecimalUtility;
public class DrillDecimalAddFuncHolder extends DrillSimpleFuncHolder{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
index 941dc49..c3a23e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java
@@ -23,11 +23,10 @@ import java.util.Map;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
-
import org.apache.drill.common.util.DecimalScalePrecisionDivideFunction;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.util.DecimalUtility;
public class DrillDecimalDivScaleFuncHolder extends DrillSimpleFuncHolder{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java
index 2cf1088..621e1dd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java
@@ -23,12 +23,11 @@ import java.util.Map;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
-
import org.apache.drill.common.util.DecimalScalePrecisionDivideFunction;
import org.apache.drill.common.util.DecimalScalePrecisionModFunction;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.util.DecimalUtility;
public class DrillDecimalModScaleFuncHolder extends DrillSimpleFuncHolder{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
index 6c04afc..cb8bfed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java
@@ -23,11 +23,10 @@ import java.util.Map;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
-
import org.apache.drill.common.util.DecimalScalePrecisionMulFunction;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.util.DecimalUtility;
public class DrillDecimalSumScaleFuncHolder extends DrillSimpleFuncHolder{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index 5ca48ce..fcc0b35 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -28,6 +28,8 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.compile.bytecode.ScalarReplacementTypes;
+import org.apache.drill.exec.compile.sig.SignatureHolder;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.BlockType;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -114,10 +116,26 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder {
return isRandom;
}
+
protected JVar[] declareWorkspaceVariables(ClassGenerator<?> g) {
JVar[] workspaceJVars = new JVar[workspaceVars.length];
for (int i = 0; i < workspaceVars.length; i++) {
- workspaceJVars[i] = g.declareClassField("work", g.getModel()._ref(workspaceVars[i].type));
+ WorkspaceReference ref = workspaceVars[i];
+ JType jtype = g.getModel()._ref(ref.type);
+
+ if(ScalarReplacementTypes.CLASSES.contains(ref.type)){
+ workspaceJVars[i] = g.declareClassField("work", jtype);
+ JBlock b = g.getBlock(SignatureHolder.DRILL_INIT_METHOD);
+ b.assign(workspaceJVars[i], JExpr._new(jtype));
+ }else{
+ workspaceJVars[i] = g.declareClassField("work", jtype);
+ }
+
+ if(ref.isInject()){
+ g.getBlock(BlockType.SETUP).assign(workspaceJVars[i], g.getMappingSet().getIncoming().invoke("getContext").invoke("getManagedBuffer"));
+ }else{
+ //g.getBlock(BlockType.SETUP).assign(workspaceJVars[i], JExpr._new(jtype));
+ }
}
return workspaceJVars;
}
@@ -159,7 +177,13 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder {
JVar[] internalVars = new JVar[workspaceJVars.length];
for (int i = 0; i < workspaceJVars.length; i++) {
- internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
+ if(decConstInputOnly){
+ internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
+ }else{
+
+ internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
+ }
+
}
Preconditions.checkNotNull(body);
@@ -303,19 +327,24 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder {
Class<?> type;
String name;
MajorType majorType;
+ boolean inject;
- public WorkspaceReference(Class<?> type, String name) {
+ public WorkspaceReference(Class<?> type, String name, boolean inject) {
super();
Preconditions.checkNotNull(type);
Preconditions.checkNotNull(name);
this.type = type;
this.name = name;
+ this.inject = inject;
}
void setMajorType(MajorType majorType) {
this.majorType = majorType;
}
+ public boolean isInject(){
+ return inject;
+ }
}
public boolean checkPrecisionRange() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 64f45c7..63ddb11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.expr.fn;
+import io.netty.buffer.DrillBuf;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
@@ -25,6 +27,8 @@ import java.net.URL;
import java.util.List;
import java.util.Map;
+import javax.inject.Inject;
+
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.expr.DrillFunc;
@@ -111,15 +115,17 @@ public class FunctionConverter {
Param param = field.getAnnotation(Param.class);
Output output = field.getAnnotation(Output.class);
Workspace workspace = field.getAnnotation(Workspace.class);
+ Inject inject = field.getAnnotation(Inject.class);
int i =0;
if(param != null) i++;
if(output != null) i++;
if(workspace != null) i++;
+ if(inject != null) i++;
if(i == 0){
- return failure("The field must be either a @Param, @Output or @Workspace field.", clazz, field);
+ return failure("The field must be either a @Param, @Output, @Inject or @Workspace field.", clazz, field);
}else if(i > 1){
- return failure("The field must be only one of @Param, @Output or @Workspace. It currently has more than one of these annotations.", clazz, field);
+ return failure("The field must be only one of @Param, @Output, @Inject or @Workspace. It currently has more than one of these annotations.", clazz, field);
}
if(param != null || output != null){
@@ -132,7 +138,7 @@ public class FunctionConverter {
// Special processing for @Output ComplexWriter
if (output != null && ComplexWriter.class.isAssignableFrom(field.getType())) {
-
+
if(outputField != null){
return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", clazz, field);
}else{
@@ -140,7 +146,7 @@ public class FunctionConverter {
}
continue;
}
-
+
// check that param and output are value holders.
if(!ValueHolder.class.isAssignableFrom(field.getType())){
return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder interface. All fields of type @Param or @Output must extend this interface..", field.getType()), clazz, field);
@@ -168,12 +174,14 @@ public class FunctionConverter {
outputField = p;
}
}
-
+
}else{
// workspace work.
- WorkspaceReference wsReference = new WorkspaceReference(field.getType(), field.getName());
+ boolean isInject = inject != null;
+ if(isInject && !field.getType().equals(DrillBuf.class)) return failure(String.format("Only DrillBuf is allowed to be injected. You attempted to inject %s.", field.getType()), clazz, field);
+ WorkspaceReference wsReference = new WorkspaceReference(field.getType(), field.getName(), isInject);
- if (template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(field.getType()) ) {
+ if (!isInject && template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(field.getType()) ) {
return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", template.name(), field.getName(), field.getType()), clazz, field);
}
@@ -229,21 +237,21 @@ public class FunctionConverter {
template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
case SIMPLE:
if (outputField.isComplexWriter)
- return new DrillComplexWriterFuncHolder(template.scope(), template.nulls(),
+ return new DrillComplexWriterFuncHolder(template.scope(), template.nulls(),
template.isBinaryCommutative(),
- template.isRandom(), registeredNames,
+ template.isRandom(), registeredNames,
ps, outputField, works, methods, imports);
else
- return new DrillSimpleFuncHolder(template.scope(), template.nulls(),
+ return new DrillSimpleFuncHolder(template.scope(), template.nulls(),
template.isBinaryCommutative(),
- template.isRandom(), registeredNames,
+ template.isRandom(), registeredNames,
ps, outputField, works, methods, imports, template.costCategory());
case SC_BOOLEAN_OPERATOR:
- return new DrillBooleanOPHolder(template.scope(), template.nulls(),
+ return new DrillBooleanOPHolder(template.scope(), template.nulls(),
template.isBinaryCommutative(),
- template.isRandom(), registeredNames,
+ template.isRandom(), registeredNames,
ps, outputField, works, methods, imports);
-
+
case DECIMAL_MAX_SCALE:
return new DrillDecimalMaxScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
new file mode 100644
index 0000000..be1d9e6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
@@ -0,0 +1,96 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.expr.fn.impl;
+
+import com.google.common.primitives.UnsignedLongs;
+
+import io.netty.util.internal.PlatformDependent;
+
+public class ByteFunctionHelpers {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
+
+ public static final int equal(final long laddr, int lStart, int lEnd, final long raddr, int rStart,
+ final int rEnd) {
+
+ int n = lEnd - lStart;
+ if (n == rEnd - rStart) {
+ long lPos = laddr + lStart;
+ long rPos = raddr + rStart;
+
+ while (n > 7) {
+ long leftLong = PlatformDependent.getLong(lPos);
+ long rightLong = PlatformDependent.getLong(rPos);
+ if (leftLong != rightLong) {
+ return 0;
+ }
+ lPos += 8;
+ rPos += 8;
+ n -= 8;
+ }
+ while (n-- != 0) {
+ byte leftByte = PlatformDependent.getByte(lPos);
+ byte rightByte = PlatformDependent.getByte(rPos);
+ if (leftByte != rightByte) {
+ return 0;
+ }
+ lPos++;
+ rPos++;
+ }
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ public static final int compare(final long laddr, int lStart, int lEnd, final long raddr, int rStart, final int rEnd) {
+ int lLen = lEnd - lStart;
+ int rLen = rEnd - rStart;
+ int n = Math.min(rLen, lLen);
+ long lPos = laddr + lStart;
+ long rPos = raddr + rStart;
+
+ while (n > 7) {
+ long leftLong = PlatformDependent.getLong(lPos);
+ long rightLong = PlatformDependent.getLong(rPos);
+ if(leftLong != rightLong){
+ return UnsignedLongs.compare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ }
+ lPos += 8;
+ rPos += 8;
+ n -= 8;
+ }
+
+ while (n-- != 0) {
+ byte leftByte = PlatformDependent.getByte(lPos);
+ byte rightByte = PlatformDependent.getByte(rPos);
+ if (leftByte != rightByte) {
+ return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+ }
+ lPos++;
+ rPos++;
+ }
+
+ if (lLen == rLen) return 0;
+
+ return lLen > rLen ? 1 : 0;
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarBinaryToDate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarBinaryToDate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarBinaryToDate.java
new file mode 100644
index 0000000..2161bfd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarBinaryToDate.java
@@ -0,0 +1,43 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionCostCategory;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.record.RecordBatch;
+
+@FunctionTemplate(names = {"castDATE", "datetype"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL, costCategory = FunctionCostCategory.COMPLEX)
+public class CastVarBinaryToDate implements DrillSimpleFunc {
+
+ @Param VarBinaryHolder in;
+ @Output DateHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getDate(in.buffer.memoryAddress(), in.start, in.end);
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java
index 12656d4..5280717 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java
@@ -17,7 +17,12 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import java.nio.charset.Charset;
+
+import javax.inject.Inject;
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -29,8 +34,6 @@ import org.apache.drill.exec.expr.holders.Var16CharHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.exec.record.RecordBatch;
-import java.nio.charset.Charset;
-
@SuppressWarnings("unused")
@FunctionTemplate(names = {"castVAR16CHAR", "to_var16char", "to_string"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL)
public class CastVarCharVar16Char implements DrillSimpleFunc {
@@ -38,19 +41,18 @@ public class CastVarCharVar16Char implements DrillSimpleFunc {
@Param BigIntHolder length;
@Output Var16CharHolder out;
@Workspace Charset charset;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Override
public void setup(RecordBatch incoming) {
charset = java.nio.charset.Charset.forName("UTF-16");
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[65536]);
}
@Override
public void eval() {
//if input length <= target_type length, do nothing
//else truncate based on target_type length
- byte[] buf = in.toString().getBytes(charset);
+ byte[] buf = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(in.start, in.end, in.buffer).getBytes(charset);
buffer.setBytes(0, buf);
out.start = 0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
index 725566f..f063466 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateTypeFunctions.java
@@ -18,7 +18,9 @@
package org.apache.drill.exec.expr.fn.impl;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import javax.inject.Inject;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -59,7 +61,7 @@ public class DateTypeFunctions {
out.months = (int) ((inputYears.value * org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths) +
(inputMonths.value));
out.days = (int) inputDays.value;
- out.milliSeconds = (int) ((inputHours.value * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
+ out.milliseconds = (int) ((inputHours.value * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
(inputMinutes.value * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) +
(inputSeconds.value * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) +
(inputMilliSeconds.value));
@@ -99,7 +101,7 @@ public class DateTypeFunctions {
public void eval() {
out.days = (int) inputDays.value;
- out.milliSeconds = (int) ((inputHours.value * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
+ out.milliseconds = (int) ((inputHours.value * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
(inputMinutes.value * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) +
(inputSeconds.value * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) +
(inputMillis.value));
@@ -174,7 +176,7 @@ public class DateTypeFunctions {
public void eval() {
- String timeZone = (inputTimeZone.toString());
+ String timeZone = (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(inputTimeZone.start, inputTimeZone.end, inputTimeZone.buffer));
out.value = ((new org.joda.time.MutableDateTime((int)inputYears.value,
(int)inputMonths.value,
(int)inputDays.value,
@@ -268,11 +270,10 @@ public class DateTypeFunctions {
@FunctionTemplate(name = "timeofday", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL, isRandom = true)
public static class TimeOfDay implements DrillSimpleFunc {
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output VarCharHolder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[100]);
}
public void eval() {
@@ -388,7 +389,7 @@ public class DateTypeFunctions {
long days = diff / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
out.months = (int) (days / org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
out.days = (int) (days % org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
- out.milliSeconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
}
}
@@ -411,7 +412,7 @@ public class DateTypeFunctions {
long days = diff / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
out.months = (int) (days / org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
out.days = (int) (days % org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
- out.milliSeconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
}
}
@@ -430,7 +431,7 @@ public class DateTypeFunctions {
long days = diff / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
out.months = (int) (days / org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
out.days = (int) (days % org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
- out.milliSeconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
}
}
@@ -453,7 +454,7 @@ public class DateTypeFunctions {
long days = diff / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
out.months = (int) (days / org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
out.days = (int) (days % org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays);
- out.milliSeconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (diff % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 7475420..7c847eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -460,7 +460,7 @@ public class HashFunctions {
int xor = 0;
for (int i = 0; i < in.nDecimalDigits; i++) {
- xor = xor ^ in.getInteger(i);
+ xor = xor ^ Decimal28SparseHolder.getInteger(i, in.start, in.buffer);
}
out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(xor).asInt();
}
@@ -480,7 +480,7 @@ public class HashFunctions {
else {
int xor = 0;
for (int i = 0; i < in.nDecimalDigits; i++) {
- xor = xor ^ in.getInteger(i);
+ xor = xor ^ NullableDecimal28SparseHolder.getInteger(i, in.start, in.buffer);
}
out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(xor).asInt();
}
@@ -499,7 +499,7 @@ public class HashFunctions {
int xor = 0;
for (int i = 0; i < in.nDecimalDigits; i++) {
- xor = xor ^ in.getInteger(i);
+ xor = xor ^ Decimal38SparseHolder.getInteger(i, in.start, in.buffer);
}
out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(xor).asInt();
}
@@ -519,7 +519,7 @@ public class HashFunctions {
else {
int xor = 0;
for (int i = 0; i < in.nDecimalDigits; i++) {
- xor = xor ^ in.getInteger(i);
+ xor = xor ^ NullableDecimal38SparseHolder.getInteger(i, in.start, in.buffer);
}
out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(xor).asInt();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleCastFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleCastFunctions.java
index 83c5540..8a09cff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleCastFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleCastFunctions.java
@@ -17,7 +17,9 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import javax.inject.Inject;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -30,8 +32,9 @@ import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.exec.record.RecordBatch;
public class SimpleCastFunctions {
- public static final ByteBuf FALSE = org.apache.drill.exec.util.ByteBufUtil.createBuffer(new byte[] {'f', 'a', 'l', 's', 'e'});
- public static final ByteBuf TRUE = org.apache.drill.exec.util.ByteBufUtil.createBuffer(new byte[] {'t', 'r', 'u', 'e'});
+ public static final byte[] TRUE = {'t','r','u','e'};
+ public static final byte[] FALSE = {'f','a','l','s','e'};
+
@FunctionTemplate(names = {"castBIT", "castBOOLEAN"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
public static class CastVarCharBoolean implements DrillSimpleFunc {
@@ -39,7 +42,9 @@ public class SimpleCastFunctions {
@Param VarCharHolder in;
@Output BitHolder out;
- public void setup(RecordBatch b) {}
+ public void setup(RecordBatch b) {
+
+ }
public void eval() {
byte[] buf = new byte[in.end - in.start];
@@ -61,14 +66,16 @@ public class SimpleCastFunctions {
@Param BitHolder in;
@Param BigIntHolder len;
@Output VarCharHolder out;
+ @Inject DrillBuf buffer;
public void setup(RecordBatch b) {}
public void eval() {
- io.netty.buffer.ByteBuf buffer = in.value == 1 ? org.apache.drill.exec.expr.fn.impl.SimpleCastFunctions.TRUE : org.apache.drill.exec.expr.fn.impl.SimpleCastFunctions.FALSE;
+ byte[] outB = in.value == 1 ? org.apache.drill.exec.expr.fn.impl.SimpleCastFunctions.TRUE : org.apache.drill.exec.expr.fn.impl.SimpleCastFunctions.FALSE;
+ buffer.setBytes(0, outB);
out.buffer = buffer;
out.start = 0;
- out.end = Math.min((int)len.value, buffer.capacity()); // truncate if target type has length smaller than that of input's string
+ out.end = Math.min((int)len.value, outB.length); // truncate if target type has length smaller than that of input's string
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
new file mode 100644
index 0000000..68f7ca4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
@@ -0,0 +1,125 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.expr.fn.impl;
+
+import io.netty.buffer.DrillBuf;
+import io.netty.util.internal.PlatformDependent;
+
+import org.joda.time.chrono.ISOChronology;
+
+import com.google.common.base.Charsets;
+
+public class StringFunctionHelpers {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StringFunctionHelpers.class);
+
+ // Assumes Alpha as [A-Za-z0-9]
+ // white space is treated as everything else.
+ public static void initCap(int start, int end, DrillBuf inBuf, DrillBuf outBuf) {
+ boolean capNext = true;
+
+ for (int id = start; id < end; id++) {
+ byte currentByte = inBuf.getByte(id);
+
+ // 'A - Z' : 0x41 - 0x5A
+ // 'a - z' : 0x61 - 0x7A
+ // '0-9' : 0x30 - 0x39
+ if (capNext) { // curCh is whitespace or first character of word.
+ if (currentByte >= 0x30 && currentByte <= 0x39) { // 0-9
+ capNext = false;
+ } else if (currentByte >= 0x41 && currentByte <= 0x5A) { // A-Z
+ capNext = false;
+ } else if (currentByte >= 0x61 && currentByte <= 0x7A) { // a-z
+ capNext = false;
+ currentByte -= 0x20; // Uppercase this character
+ }
+ // else {} whitespace
+ } else { // Inside of a word or white space after end of word.
+ if (currentByte >= 0x30 && currentByte <= 0x39) { // 0-9
+ // noop
+ } else if (currentByte >= 0x41 && currentByte <= 0x5A) { // A-Z
+ currentByte -= 0x20; // Lowercase this character
+ } else if (currentByte >= 0x61 && currentByte <= 0x7A) { // a-z
+ // noop
+ } else { // whitespace
+ capNext = true;
+ }
+ }
+
+ outBuf.setByte(id, currentByte);
+ } // end of for_loop
+ }
+
+ public static String toStringFromUTF8(int start, int end, DrillBuf buffer) {
+ byte[] buf = new byte[end - start];
+ buffer.getBytes(start, buf, 0, end - start);
+ String s = new String(buf, Charsets.UTF_8);
+ return s;
+ }
+
+ public static String toStringFromUTF16(int start, int end, DrillBuf buffer) {
+ byte[] buf = new byte[end - start];
+ buffer.getBytes(start, buf, 0, end - start);
+ return new String(buf, Charsets.UTF_16);
+ }
+
+ private static final ISOChronology CHRONOLOGY = org.joda.time.chrono.ISOChronology.getInstanceUTC();
+
+ public static long getDate(long memoryAddress, int start, int end){
+ long index = memoryAddress + start;
+ final long endIndex = memoryAddress + end;
+ int digit = 0;
+ int radix = 10; // Base 10 digits
+
+ // Stores three fields (year, month, day)
+ int[] dateFields = new int[3];
+ int dateIndex = 0;
+ int value = 0;
+
+ while (dateIndex < 3 && index < endIndex) {
+ digit = Character.digit(PlatformDependent.getByte(index++), radix);
+
+ if (digit == -1) {
+ dateFields[dateIndex++] = value;
+ value = 0;
+ } else {
+ value = (value * 10) + digit;
+ }
+ }
+
+ if (dateIndex < 3) {
+ // If we reached the end of input, we would have not encountered a separator, store the last value
+ dateFields[dateIndex++] = value;
+ }
+
+ /* Handle two digit years
+ * Follow convention as done by Oracle, Postgres
+ * If range of two digits is between 70 - 99 then year = 1970 - 1999
+ * Else if two digits is between 00 - 69 = 2000 - 2069
+ */
+ if (dateFields[0] < 100) {
+ if (dateFields[0] < 70) {
+ dateFields[0] += 2000;
+ } else {
+ dateFields[0] += 1900;
+ }
+ }
+
+ return CHRONOLOGY.getDateTimeMillis(dateFields[0], dateFields[1], dateFields[2], 0);
+ }
+}
[32/47] git commit: Bundle the HBase 0.94 jar compiled with Hadoop 2
profile
Posted by ja...@apache.org.
Bundle the HBase 0.94 jar compiled with Hadoop 2 profile
+ Removed the HBase shaded jar module.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c4cc9e4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c4cc9e4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c4cc9e4a
Branch: refs/heads/master
Commit: c4cc9e4a877fd73b2f8c3331720e1a69d7b8074f
Parents: a99aeca
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Wed Aug 27 23:35:07 2014 +0530
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 13:33:47 2014 -0700
----------------------------------------------------------------------
contrib/hbase-shaded/pom.xml | 373 ---------------------------------
contrib/pom.xml | 1 -
distribution/pom.xml | 110 +++++++++-
distribution/src/assemble/bin.xml | 1 -
4 files changed, 99 insertions(+), 386 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c4cc9e4a/contrib/hbase-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/hbase-shaded/pom.xml b/contrib/hbase-shaded/pom.xml
deleted file mode 100644
index cc61291..0000000
--- a/contrib/hbase-shaded/pom.xml
+++ /dev/null
@@ -1,373 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.drill.contrib</groupId>
- <artifactId>drill-contrib-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>drill-hbase-shaded</artifactId>
- <packaging>jar</packaging>
- <name>contrib/hbase-shaded</name>
-
- <properties>
- <hbase.TestSuite>**/HBaseTestsSuite.class</hbase.TestSuite>
- </properties>
-
- <profiles>
- <profile>
- <id>default-hadoop</id>
- <activation>
- <property>
- <name>!alt-hadoop</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>0.94.11</version>
- <exclusions>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>asm</artifactId>
- <groupId>asm</groupId>
- </exclusion>
- <exclusion>
- <artifactId>libthrift</artifactId>
- <groupId>org.apache.thrift</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jruby-complete</artifactId>
- <groupId>org.jruby</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <!-- contrary to other Drill modules, this HBase storage module
- compiles against hadoop 1.x to allow running a mini HBase
- cluster to run the unit tests.
- This will change once we move to HBase 0.98 or later.
- -->
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>1.2.1</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- <exclusion>
- <artifactId>asm</artifactId>
- <groupId>asm</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.1</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <includes>
- <include>org.apache.hbase:hbase</include>
- <include>org.apache.hadoop:hadoop-core</include>
- </includes>
- </artifactSet>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <relocations>
- <relocation>
- <pattern>org.apache.hadoop.net</pattern>
- <shadedPattern>org.apache.hadoop1.net</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>mapr</id>
- <properties>
- <alt-hadoop>mapr</alt-hadoop>
- <rat.excludeSubprojects>true</rat.excludeSubprojects>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>0.94.17-mapr-1405-m7-4.0.0-FCS</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>asm</artifactId>
- <groupId>asm</groupId>
- </exclusion>
- <exclusion>
- <artifactId>libthrift</artifactId>
- <groupId>org.apache.thrift</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jruby-complete</artifactId>
- <groupId>org.jruby</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-core</artifactId>
- <groupId>org.apache</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>1.0.3-mapr-3.0.2</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <classifier>tests</classifier>
- <version>0.94.13-mapr-1401-m7-3.0.2</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>asm</artifactId>
- <groupId>asm</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.1</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <filters>
- <filter>
- <artifact>org.apache.hadoop:hadoop-core</artifact>
- <excludes>
- <exclude>org.apache.hadoop.fs.*</exclude>
- </excludes>
- </filter>
- </filters>
- <artifactSet>
- <includes>
- <include>org.apache.hbase:hbase</include>
- <include>org.apache.hadoop:hadoop-core</include>
- </includes>
- </artifactSet>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <relocations>
- <relocation>
- <pattern>org.apache.hadoop.net</pattern>
- <shadedPattern>org.apache.hadoop1.net</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <includes>
- <include>${hbase.TestSuite}</include>
- </includes>
- <systemProperties>
- <property>
- <name>hbase.test.root</name>
- <value>${project.build.directory}/data</value>
- </property>
- <property>
- <name>logback.log.dir</name>
- <value>${project.build.directory}/surefire-reports</value>
- </property>
- </systemProperties>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.1</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <includes>
- <include>org.apache.hbase:hbase</include>
- <include>org.apache.hadoop:hadoop-core</include>
- </includes>
- </artifactSet>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <relocations>
- <relocation>
- <pattern>org.apache.hadoop.net</pattern>
- <shadedPattern>org.apache.hadoop1.net</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>unpack</id>
- <phase>package</phase>
- <goals>
- <goal>unpack</goal>
- </goals>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>org.apache.drill.contrib</groupId>
- <artifactId>drill-hbase-shaded</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <overWrite>true</overWrite>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- <includes>**/**</includes>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c4cc9e4a/contrib/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/pom.xml b/contrib/pom.xml
index 0c979e9..de7709d 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -32,7 +32,6 @@
</dependencies>
<modules>
- <module>hbase-shaded</module>
<module>storage-hbase</module>
<module>storage-hive</module>
<module>sqlline</module>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c4cc9e4a/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 0d493ce..069f3ac 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -112,24 +112,112 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.drill.contrib</groupId>
- <artifactId>drill-hbase-shaded</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.22-drill-r1</version>
<exclusions>
<exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>libthrift</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jruby-complete</artifactId>
+ <groupId>org.jruby</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>avro-ipc</artifactId>
+ <groupId>org.apache.avro</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>avro</artifactId>
+ <groupId>org.apache.avro</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-codec</artifactId>
+ <groupId>commons-codec</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jasper-runtime</artifactId>
+ <groupId>tomcat</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-core-asl</artifactId>
+ <groupId>org.codehaus.jackson</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <groupId>org.codehaus.jackson</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jasper-compiler</artifactId>
+ <groupId>tomcat</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jamon-runtime</artifactId>
+ <groupId>org.jamon</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jaxb-api</artifactId>
+ <groupId>javax.xml.bind</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>zookeeper</artifactId>
+ <groupId>org.apache.zookeeper</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jsp-2.1</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jsp-api-2.1</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
</exclusion>
<exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
+ <artifactId>stax-api</artifactId>
+ <groupId>stax</groupId>
</exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c4cc9e4a/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index 68b58bf..6d7fb6b 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -49,7 +49,6 @@
<include>org.apache.drill.contrib.storage-hive:drill-hive-exec-shaded</include>
<include>org.apache.drill.contrib.data:tpch-sample-data:jar</include>
<include>org.apache.drill.contrib:drill-storage-hbase</include>
- <include>org.apache.drill.contrib:drill-hbase-shaded</include>
</includes>
<excludes>
<exclude>org.apache.drill.contrib.storage-hive:drill-storage-hive-core:jar:tests</exclude>
[38/47] git commit: Update RPC version number to correspond to
changes in nullable buffer from bitmap to bytes.
Posted by ja...@apache.org.
Update RPC version number to correspond to changes in nullable buffer from bitmap to bytes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/496b5d0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/496b5d0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/496b5d0a
Branch: refs/heads/master
Commit: 496b5d0a1a8ce5e1cdaf06d127400f1e7e7687b0
Parents: d944918
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Aug 27 17:31:59 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 17:32:39 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/rpc/data/DataRpcConfig.java | 8 ++++----
.../java/org/apache/drill/exec/rpc/user/UserRpcConfig.java | 6 +++---
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/496b5d0a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcConfig.java
index 6224827..45d95d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcConfig.java
@@ -30,13 +30,13 @@ import org.apache.drill.exec.rpc.RpcConfig;
public class DataRpcConfig {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataRpcConfig.class);
-
+
public static RpcConfig MAPPING = RpcConfig.newBuilder("BIT-DATA-RPC-MAPPING") //
.add(RpcType.HANDSHAKE, BitClientHandshake.class, RpcType.HANDSHAKE, BitServerHandshake.class)
.add(RpcType.REQ_RECORD_BATCH, FragmentRecordBatch.class, RpcType.ACK, Ack.class)
.build();
-
- public static int RPC_VERSION = 1;
-
+
+ public static int RPC_VERSION = 2;
+
public static final Response OK = new Response(RpcType.ACK, Acks.OK);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/496b5d0a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcConfig.java
index be5bb8b..56389b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcConfig.java
@@ -28,13 +28,13 @@ import org.apache.drill.exec.rpc.RpcConfig;
public class UserRpcConfig {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserRpcConfig.class);
-
+
public static RpcConfig MAPPING = RpcConfig.newBuilder("USER-RPC-MAPPING") //
.add(RpcType.HANDSHAKE, UserToBitHandshake.class, RpcType.HANDSHAKE, BitToUserHandshake.class) //user to bit.
.add(RpcType.RUN_QUERY, RunQuery.class, RpcType.QUERY_HANDLE, QueryId.class) //user to bit
.add(RpcType.CANCEL_QUERY, QueryId.class, RpcType.ACK, Ack.class) //user to bit
.add(RpcType.QUERY_RESULT, QueryResult.class, RpcType.ACK, Ack.class) //bit to user
.build();
-
- public static int RPC_VERSION = 1;
+
+ public static int RPC_VERSION = 2;
}
[12/47] git commit: DRILL-1310: Fix assertion in ProjectRecordBatch
for certain types of star queries.
Posted by ja...@apache.org.
DRILL-1310: Fix assertion in ProjectRecordBatch for certain types of star queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/de05070c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/de05070c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/de05070c
Branch: refs/heads/master
Commit: de05070cc41271ba12ae5bfede27ab12993be564
Parents: 2275da9
Author: Aman Sinha <as...@maprtech.com>
Authored: Mon Aug 18 16:32:20 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 12:04:05 2014 -0700
----------------------------------------------------------------------
.../impl/project/ProjectRecordBatch.java | 50 ++++++++++++++++----
1 file changed, 40 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/de05070c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index a7e1360..c8929d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -317,9 +317,12 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
String outputName = getRef(namedExpression).getRootSegment().getPath();
if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
- // TODO: Revisit this logic. It is possible that the output names array does not contain the expression of interest in index 0
- if (result.outputNames.get(0) == EMPTY_STRING) continue;
- outputName = result.outputNames.get(0);
+ for (int j = 0; j < result.outputNames.size(); j++) {
+ if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
+ outputName = result.outputNames.get(j);
+ break;
+ }
+ }
}
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
@@ -460,18 +463,19 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
boolean exprHasPrefix = expr.getPath().contains(StarColumnHelper.PREFIX_DELIMITER);
boolean refHasPrefix = ref.getPath().contains(StarColumnHelper.PREFIX_DELIMITER);
boolean exprIsStar = expr.getPath().equals(StarColumnHelper.STAR_COLUMN);
- boolean refIsStar = ref.getPath().equals(StarColumnHelper.STAR_COLUMN);
boolean refContainsStar = ref.getPath().contains(StarColumnHelper.STAR_COLUMN);
boolean exprContainsStar = expr.getPath().contains(StarColumnHelper.STAR_COLUMN);
boolean refEndsWithStar = ref.getPath().endsWith(StarColumnHelper.STAR_COLUMN);
String exprPrefix = EMPTY_STRING;
+ String exprSuffix = expr.getPath();
if (exprHasPrefix) {
// get the prefix of the expr
String[] exprComponents = expr.getPath().split(StarColumnHelper.PREFIX_DELIMITER, 2);
assert(exprComponents.length == 2);
exprPrefix = exprComponents[0];
+ exprSuffix = exprComponents[1];
result.prefix = exprPrefix;
}
@@ -568,14 +572,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
String newName = ref.getPath();
addToResultMaps(newName, result, false);
}
- // input has prefix but output does not; this would happen for a top-level project
+ // input has prefix but output does not
else if (exprHasPrefix && !refHasPrefix) {
int k = 0;
result.outputNames = Lists.newArrayListWithCapacity(incomingSchemaSize);
for (int j=0; j < incomingSchemaSize; j++) {
result.outputNames.add(EMPTY_STRING); // initialize
}
- // remove the prefix from the incoming column names
+
for(VectorWrapper<?> wrapper : incoming) {
ValueVector vvIn = wrapper.getValueVector();
String name = vvIn.getField().getPath().getRootSegment().getPath();
@@ -587,8 +591,15 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
String namePrefix = components[0];
String nameSuffix = components[1];
if (exprPrefix.equals(namePrefix)) {
- String newName = getUniqueName(nameSuffix, result); // for top level we need to make names unique
- result.outputNames.set(k, newName);
+ if (refContainsStar) {
+ // remove the prefix from the incoming column names
+ String newName = getUniqueName(nameSuffix, result); // for top level we need to make names unique
+ result.outputNames.set(k, newName);
+ } else if (exprSuffix.equals(nameSuffix)) {
+ // example: ref: $f1, expr: T0<PREFIX><column_name>
+ String newName = ref.getPath();
+ result.outputNames.set(k, newName);
+ }
} else {
result.outputNames.add(EMPTY_STRING);
}
@@ -599,11 +610,30 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
else if (exprHasPrefix && refHasPrefix) {
String[] input = expr.getPath().split(StarColumnHelper.PREFIX_DELIMITER, 2);
assert(input.length == 2);
- String inputSuffix = input[1];
assert false : "Unexpected project expression or reference"; // not handled yet
}
else {
- assert false : "Unexpected project expression or reference." ;
+ // if the incoming schema's column name matches the expression name of the Project,
+ // then we just want to pick the ref name as the output column name
+ result.outputNames = Lists.newArrayListWithCapacity(incomingSchemaSize);
+ for (int j=0; j < incomingSchemaSize; j++) {
+ result.outputNames.add(EMPTY_STRING); // initialize
+ }
+
+ int k = 0;
+ for(VectorWrapper<?> wrapper : incoming) {
+ ValueVector vvIn = wrapper.getValueVector();
+ String incomingName = vvIn.getField().getPath().getRootSegment().getPath();
+
+ if (expr.getPath().equals(incomingName)) {
+ String newName = ref.getPath();
+ if (!result.outputMap.containsKey(newName)) {
+ result.outputNames.set(k, newName);
+ result.outputMap.put(newName, newName);
+ }
+ }
+ k++;
+ }
}
}
[25/47] git commit: Switch to DrillBuf Add @Inject DrillBuf Move
comparison functions to memory sensitive ones Add scalar replacement
functionality for value holders Simplify date parsing function Add local
compiled code caching
Posted by ja...@apache.org.
Switch to DrillBuf
Add @Inject DrillBuf
Move comparison functions to memory sensitive ones
Add scalar replacement functionality for value holders
Simplify date parsing function
Add local compiled code caching
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5c5cef06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5c5cef06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5c5cef06
Branch: refs/heads/master
Commit: 5c5cef06881c0ee8d1c2b203e166dffac69bfd14
Parents: fb93576
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Jul 9 14:01:35 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 21:00:43 2014 -0700
----------------------------------------------------------------------
.../expression/ExpressionStringBuilder.java | 19 +-
.../common/expression/ValueExpressions.java | 152 ++--
.../drill/common/util/CoreDecimalUtility.java | 76 ++
.../DecimalScalePrecisionDivideFunction.java | 3 +-
.../util/DecimalScalePrecisionModFunction.java | 3 +-
.../drill/common/util/DecimalUtility.java | 735 ----------------
.../codegen/templates/ObjectInspectors.java | 2 +-
exec/java-exec/pom.xml | 18 +-
exec/java-exec/src/main/codegen/data/Casts.tdd | 2 +-
.../src/main/codegen/data/ValueVectorTypes.tdd | 66 +-
.../src/main/codegen/includes/vv_imports.ftl | 4 +-
.../main/codegen/templates/CastDateVarChar.java | 7 +-
.../main/codegen/templates/CastFunctions.java | 2 +
.../templates/CastFunctionsSrcVarLen.java | 2 +
.../CastFunctionsSrcVarLenTargetVarLen.java | 2 +
.../templates/CastFunctionsTargetVarLen.java | 8 +-
.../codegen/templates/CastIntervalInterval.java | 6 +-
.../codegen/templates/CastIntervalVarChar.java | 26 +-
.../main/codegen/templates/CastVarCharDate.java | 11 +-
.../codegen/templates/CastVarCharInterval.java | 6 +-
.../codegen/templates/ComparisonFunctions.java | 46 +-
.../templates/ConvertToNullableHolder.java | 6 +-
.../templates/DateIntervalAggrFunctions1.java | 28 +-
.../DateDateArithmeticFunctions.java | 6 +-
.../DateIntervalArithmeticFunctions.java | 6 +-
.../DateToCharFunctions.java | 7 +-
.../DateIntervalFunctionTemplates/Extract.java | 12 +-
.../IntervalIntervalArithmetic.java | 12 +-
.../IntervalNumericArithmetic.java | 10 +-
.../ToTimeStampFunction.java | 2 +-
.../templates/DateIntervalFunctions.java | 28 +-
.../Decimal/CastDecimalDenseDecimalSparse.java | 34 +-
.../templates/Decimal/CastDecimalFloat.java | 12 +-
.../templates/Decimal/CastDecimalInt.java | 18 +-
.../templates/Decimal/CastDecimalSimilar.java | 22 +-
.../Decimal/CastDecimalSparseDecimalDense.java | 38 +-
.../templates/Decimal/CastDecimalVarchar.java | 51 +-
.../templates/Decimal/CastDownwardDecimal.java | 44 +-
.../templates/Decimal/CastFloatDecimal.java | 20 +-
.../templates/Decimal/CastIntDecimal.java | 26 +-
.../templates/Decimal/CastSrcDecimalSimple.java | 57 +-
.../templates/Decimal/CastVarCharDecimal.java | 58 +-
.../Decimal/DecimalAggrTypeFunctions1.java | 51 +-
.../Decimal/DecimalAggrTypeFunctions2.java | 14 +-
.../templates/Decimal/DecimalFunctions.java | 348 ++++----
.../codegen/templates/FixedValueVectors.java | 221 +++--
.../codegen/templates/HolderReaderImpl.java | 31 +-
.../templates/IntervalAggrFunctions2.java | 8 +-
.../templates/JsonOutputRecordWriter.java | 2 +-
.../codegen/templates/NullableValueVectors.java | 106 ++-
.../templates/NumericToCharFunctions.java | 12 +-
.../templates/ParquetOutputRecordWriter.java | 4 +-
.../codegen/templates/RepeatedValueVectors.java | 18 +-
.../main/codegen/templates/SqlAccessors.java | 2 +-
.../main/codegen/templates/ValueHolders.java | 164 ++--
.../templates/VarCharAggrFunctions1.java | 56 +-
.../templates/VariableLengthVectors.java | 72 +-
.../java/io/netty/buffer/AccountingByteBuf.java | 866 -------------------
.../src/main/java/io/netty/buffer/DrillBuf.java | 674 +++++++++++++++
.../java/io/netty/buffer/FakeAllocator.java | 137 +++
.../netty/buffer/PooledByteBufAllocatorL.java | 2 +-
.../netty/buffer/UnsafeDirectLittleEndian.java | 24 +-
.../cache/VectorAccessibleSerializable.java | 5 +-
.../drill/exec/compile/ByteCodeLoader.java | 4 +-
.../drill/exec/compile/ClassTransformer.java | 103 +--
.../apache/drill/exec/compile/CodeCompiler.java | 89 ++
.../exec/compile/DrillInitMethodVisitor.java | 44 +
.../apache/drill/exec/compile/MergeAdapter.java | 110 ++-
.../exec/compile/TemplateClassDefinition.java | 17 +-
.../exec/compile/bytecode/DirectSorter.java | 33 +
.../compile/bytecode/InstructionModifier.java | 259 ++++++
.../compile/bytecode/ReplacingBasicValue.java | 57 ++
.../compile/bytecode/ReplacingInterpreter.java | 91 ++
.../compile/bytecode/ScalarReplacementNode.java | 66 ++
.../bytecode/ScalarReplacementTypes.java | 117 +++
.../bytecode/TrackingInstructionList.java | 69 ++
.../exec/compile/bytecode/ValueHolderIden.java | 223 +++++
.../bytecode/ValueHolderReplacementVisitor.java | 75 ++
.../exec/compile/sig/CodeGeneratorMethod.java | 18 +-
.../exec/compile/sig/GeneratorMapping.java | 12 +-
.../drill/exec/compile/sig/MappingSet.java | 4 +
.../drill/exec/compile/sig/SignatureHolder.java | 39 +-
.../apache/drill/exec/expr/ClassGenerator.java | 6 +-
.../apache/drill/exec/expr/CodeGenerator.java | 61 +-
.../drill/exec/expr/EvaluationVisitor.java | 138 ++-
.../drill/exec/expr/GetSetVectorHelper.java | 175 ++++
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 70 +-
.../exec/expr/fn/DrillDecimalAddFuncHolder.java | 3 +-
.../expr/fn/DrillDecimalDivScaleFuncHolder.java | 3 +-
.../expr/fn/DrillDecimalModScaleFuncHolder.java | 3 +-
.../expr/fn/DrillDecimalSumScaleFuncHolder.java | 3 +-
.../drill/exec/expr/fn/DrillFuncHolder.java | 35 +-
.../drill/exec/expr/fn/FunctionConverter.java | 36 +-
.../exec/expr/fn/impl/ByteFunctionHelpers.java | 96 ++
.../exec/expr/fn/impl/CastVarBinaryToDate.java | 43 +
.../exec/expr/fn/impl/CastVarCharVar16Char.java | 14 +-
.../exec/expr/fn/impl/DateTypeFunctions.java | 21 +-
.../drill/exec/expr/fn/impl/HashFunctions.java | 8 +-
.../exec/expr/fn/impl/SimpleCastFunctions.java | 19 +-
.../expr/fn/impl/StringFunctionHelpers.java | 125 +++
.../exec/expr/fn/impl/StringFunctions.java | 193 ++---
.../expr/fn/impl/conv/BigIntBEConvertTo.java | 9 +-
.../exec/expr/fn/impl/conv/BigIntConvertTo.java | 8 +-
.../expr/fn/impl/conv/BigIntVLongConvertTo.java | 8 +-
.../expr/fn/impl/conv/BooleanByteConvertTo.java | 8 +-
.../expr/fn/impl/conv/DateEpochBEConvertTo.java | 8 +-
.../expr/fn/impl/conv/DateEpochConvertTo.java | 8 +-
.../exec/expr/fn/impl/conv/DoubleConvertTo.java | 8 +-
.../exec/expr/fn/impl/conv/FloatConvertTo.java | 8 +-
.../exec/expr/fn/impl/conv/IntBEConvertTo.java | 8 +-
.../exec/expr/fn/impl/conv/IntConvertTo.java | 8 +-
.../expr/fn/impl/conv/IntVIntConvertTo.java | 8 +-
.../exec/expr/fn/impl/conv/JsonConvertFrom.java | 20 +-
.../exec/expr/fn/impl/conv/JsonConvertTo.java | 25 +-
.../expr/fn/impl/conv/SmallIntBEConvertTo.java | 8 +-
.../expr/fn/impl/conv/SmallIntConvertTo.java | 8 +-
.../expr/fn/impl/conv/TimeEpochBEConvertTo.java | 8 +-
.../expr/fn/impl/conv/TimeEpochConvertTo.java | 8 +-
.../expr/fn/impl/conv/TinyIntConvertTo.java | 8 +-
.../exec/expr/fn/impl/conv/UInt8ConvertTo.java | 8 +-
.../org/apache/drill/exec/memory/Accountor.java | 10 +-
.../drill/exec/memory/BufferAllocator.java | 11 +-
.../drill/exec/memory/TopLevelAllocator.java | 59 +-
.../apache/drill/exec/ops/FragmentContext.java | 39 +-
.../apache/drill/exec/ops/OperatorContext.java | 29 +
.../drill/exec/physical/impl/OutputMutator.java | 8 +-
.../drill/exec/physical/impl/ScanBatch.java | 30 +-
.../impl/aggregate/HashAggTemplate.java | 47 +-
.../exec/physical/impl/common/HashTable.java | 10 +-
.../physical/impl/common/HashTableTemplate.java | 68 +-
.../exec/physical/impl/common/IndexPointer.java | 24 +
.../exec/physical/impl/join/HashJoinBatch.java | 15 +-
.../planner/sql/handlers/DefaultSqlHandler.java | 1 +
.../org/apache/drill/exec/record/DeadBuf.java | 19 +-
.../drill/exec/record/RawFragmentBatch.java | 7 +-
.../drill/exec/record/RecordBatchLoader.java | 7 +-
.../apache/drill/exec/record/TypedFieldId.java | 9 +
.../apache/drill/exec/record/WritableBatch.java | 42 +-
.../exec/record/selection/SelectionVector2.java | 23 +-
.../drill/exec/resolver/TypeCastRules.java | 2 +-
.../drill/exec/rpc/ProtobufLengthDecoder.java | 11 +-
.../exec/rpc/data/DataResponseHandler.java | 5 +-
.../exec/rpc/data/DataResponseHandlerImpl.java | 4 +-
.../apache/drill/exec/rpc/data/DataServer.java | 6 +-
.../drill/exec/rpc/user/QueryResultBatch.java | 25 +-
.../drill/exec/rpc/user/QueryResultHandler.java | 3 +-
.../drill/exec/server/DrillbitContext.java | 11 +-
.../server/options/SystemOptionManager.java | 3 +-
.../exec/store/easy/json/JSONRecordReader2.java | 6 +-
.../columnreaders/FixedByteAlignedReader.java | 13 +-
.../NullableFixedByteAlignedReaders.java | 20 +-
.../NullableVarLengthValuesColumn.java | 8 +-
.../store/parquet/columnreaders/PageReader.java | 12 +-
.../columnreaders/VarLengthColumnReaders.java | 77 +-
.../columnreaders/VarLengthValuesColumn.java | 8 +-
.../parquet2/DrillParquetGroupConverter.java | 59 +-
.../exec/store/parquet2/DrillParquetReader.java | 2 +-
.../DrillParquetRecordMaterializer.java | 11 +-
.../drill/exec/store/pojo/PojoRecordReader.java | 47 +-
.../apache/drill/exec/store/pojo/Writers.java | 24 +-
.../org/apache/drill/exec/util/ByteBufUtil.java | 45 +-
.../apache/drill/exec/util/DecimalUtility.java | 692 +++++++++++++++
.../drill/exec/vector/BaseDataValueVector.java | 27 +-
.../drill/exec/vector/BaseValueVector.java | 2 +-
.../org/apache/drill/exec/vector/BitVector.java | 11 +-
.../drill/exec/vector/FixedWidthVector.java | 12 +-
.../apache/drill/exec/vector/ObjectVector.java | 6 +-
.../exec/vector/RepeatedFixedWidthVector.java | 6 +-
.../vector/RepeatedVariableWidthVector.java | 4 +-
.../drill/exec/vector/ValueHolderHelper.java | 43 +-
.../apache/drill/exec/vector/ValueVector.java | 10 +-
.../drill/exec/vector/VariableWidthVector.java | 12 +-
.../apache/drill/exec/vector/VectorTrimmer.java | 4 +-
.../drill/exec/vector/complex/MapVector.java | 12 +-
.../exec/vector/complex/RepeatedListVector.java | 8 +-
.../exec/vector/complex/RepeatedMapVector.java | 14 +-
.../exec/vector/complex/fn/JsonReader.java | 31 +-
.../vector/complex/fn/JsonReaderWithState.java | 8 +-
.../exec/work/batch/SpoolingRawBatchBuffer.java | 7 +-
.../java/org/apache/drill/exec/RunRootExec.java | 74 ++
.../apache/drill/exec/client/DumpCatTest.java | 4 +
.../exec/compile/TestClassTransformation.java | 9 +-
.../exec/compile/TestEvaluationVisitor.java | 2 +-
.../compile/bytecode/ReplaceMethodInvoke.java | 105 +++
.../apache/drill/exec/expr/ExpressionTest.java | 2 +-
.../fn/impl/TestByteComparisonFunctions.java | 129 +++
.../drill/exec/fn/impl/TestMathFunctions.java | 2 +
.../exec/fn/impl/TestNewMathFunctions.java | 2 +
.../exec/fn/impl/TestRepeatedFunction.java | 2 +
.../exec/physical/impl/TestCastFunctions.java | 14 +-
.../physical/impl/TestComparisonFunctions.java | 2 +
.../physical/impl/TestConvertFunctions.java | 6 +-
.../impl/TestImplicitCastFunctions.java | 2 +
.../exec/physical/impl/TestOptiqPlans.java | 1 +
.../exec/physical/impl/TestSimpleFunctions.java | 11 +-
.../exec/physical/impl/TestStringFunctions.java | 8 +-
.../drill/exec/physical/impl/agg/TestAgg.java | 2 +
.../physical/impl/common/TestHashTable.java | 2 +
.../physical/impl/filter/TestSimpleFilter.java | 3 +
.../exec/physical/impl/join/TestHashJoin.java | 6 +-
.../exec/physical/impl/join/TestMergeJoin.java | 8 +-
.../physical/impl/limit/TestSimpleLimit.java | 4 +
.../impl/project/TestSimpleProjection.java | 2 +
.../exec/physical/impl/sort/TestSimpleSort.java | 3 +
.../physical/impl/svremover/TestSVRemover.java | 2 +
.../impl/trace/TestTraceMultiRecordBatch.java | 6 +
.../impl/trace/TestTraceOutputDump.java | 2 +
.../physical/impl/union/TestSimpleUnion.java | 2 +
.../drill/exec/record/vector/TestLoad.java | 3 +-
.../apache/drill/exec/server/TestBitRpc.java | 3 +-
.../drill/exec/store/TestOutputMutator.java | 7 +
.../store/parquet/ParquetRecordReaderTest.java | 9 +
.../vector/complex/writer/TestJsonReader.java | 7 +-
exec/java-exec/src/test/sh/runexec | 33 +
214 files changed, 5900 insertions(+), 3522 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
index 0e778c5..a15536e 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -17,36 +17,29 @@
*/
package org.apache.drill.common.expression;
-import java.io.IOException;
+import java.math.BigDecimal;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.joda.time.Period;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.DateTimeFormatterBuilder;
-import org.joda.time.format.DateTimeParser;
import com.google.common.collect.ImmutableList;
-import org.apache.drill.common.util.DecimalUtility;
-
-import java.math.BigDecimal;
public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class);
@@ -76,7 +69,7 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
sb.append(") ");
return null;
}
-
+
@Override
public Void visitBooleanOperator(BooleanOperator op, StringBuilder sb) throws RuntimeException {
return visitFunctionCall(op, sb);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index a1f69c2..c4237d5 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.common.expression;
-import java.util.GregorianCalendar;
import java.math.BigDecimal;
+import java.util.GregorianCalendar;
import java.util.Iterator;
import org.apache.drill.common.expression.visitors.ExprVisitor;
@@ -26,7 +26,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.common.util.CoreDecimalUtility;
import com.google.common.collect.Iterators;
@@ -96,7 +96,7 @@ public class ValueExpressions {
}
public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) {
- String numStr = (sign == null) ? s : sign+s;
+ String numStr = (sign == null) ? s : sign+s;
try {
int a = Integer.parseInt(numStr);
return new IntExpression(a, ep);
@@ -138,12 +138,12 @@ public class ValueExpressions {
}
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
@@ -208,12 +208,12 @@ public class ValueExpressions {
}
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
@@ -248,17 +248,17 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
@@ -272,7 +272,7 @@ public class ValueExpressions {
super(pos);
this.scale = input.scale();
this.precision = input.precision();
- this.decimal = DecimalUtility.getDecimal9FromBigDecimal(input, scale, precision);
+ this.decimal = CoreDecimalUtility.getDecimal9FromBigDecimal(input, scale, precision);
}
@@ -302,17 +302,17 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
@@ -326,7 +326,7 @@ public class ValueExpressions {
super(pos);
this.scale = input.scale();
this.precision = input.precision();
- this.decimal = DecimalUtility.getDecimal18FromBigDecimal(input, scale, precision);
+ this.decimal = CoreDecimalUtility.getDecimal18FromBigDecimal(input, scale, precision);
}
@@ -356,14 +356,14 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
@@ -397,14 +397,14 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
}
@@ -438,15 +438,15 @@ public class ValueExpressions {
}
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
@@ -478,17 +478,17 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
public static class LongExpression extends LogicalExpressionBase {
@@ -526,15 +526,15 @@ public class ValueExpressions {
}
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
@@ -572,14 +572,14 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
@@ -621,15 +621,15 @@ public class ValueExpressions {
}
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
public static class TimeStampExpression extends LogicalExpressionBase {
@@ -665,14 +665,14 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
@@ -711,14 +711,14 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
}
-
+
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
@@ -766,15 +766,15 @@ public class ValueExpressions {
}
@Override
- public int getSelfCost() {
- return 0; // TODO
+ public int getSelfCost() {
+ return 0; // TODO
}
-
+
@Override
- public int getCumulativeCost() {
+ public int getCumulativeCost() {
return 0; // TODO
}
-
+
}
public static class QuotedString extends ValueExpression<String> {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
new file mode 100644
index 0000000..c5cd8c0
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import java.math.BigDecimal;
+
+import org.apache.drill.common.types.TypeProtos;
+
+public class CoreDecimalUtility {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class);
+
+ public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+ // Truncate or pad to set the input to the correct scale
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ return (input.unscaledValue().longValue());
+ }
+
+ public static int getMaxPrecision(TypeProtos.MinorType decimalType) {
+ if (decimalType == TypeProtos.MinorType.DECIMAL9) {
+ return 9;
+ } else if (decimalType == TypeProtos.MinorType.DECIMAL18) {
+ return 18;
+ } else if (decimalType == TypeProtos.MinorType.DECIMAL28SPARSE) {
+ return 28;
+ } else if (decimalType == TypeProtos.MinorType.DECIMAL38SPARSE) {
+ return 38;
+ }
+ return 0;
+ }
+
+ /*
+ * Function returns the Minor decimal type given the precision
+ */
+ public static TypeProtos.MinorType getDecimalDataType(int precision) {
+ if (precision <= 9) {
+ return TypeProtos.MinorType.DECIMAL9;
+ } else if (precision <= 18) {
+ return TypeProtos.MinorType.DECIMAL18;
+ } else if (precision <= 28) {
+ return TypeProtos.MinorType.DECIMAL28SPARSE;
+ } else {
+ return TypeProtos.MinorType.DECIMAL38SPARSE;
+ }
+ }
+
+ /*
+ * Given a precision it provides the max precision of that decimal data type;
+ * For eg: given the precision 12, we would use DECIMAL18 to store the data
+ * which has a max precision range of 18 digits
+ */
+ public static int getPrecisionRange(int precision) {
+ return getMaxPrecision(getDecimalDataType(precision));
+ }
+ public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
+ // Truncate/ or pad to set the input to the correct scale
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ return (input.unscaledValue().intValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
index 4d3b939..6214e3e 100644
--- a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.util;
+
/*
* Here we compute the scale and precision of the output decimal data type
* based on the input scale and precision. Since division operation can be
@@ -51,7 +52,7 @@ public class DecimalScalePrecisionDivideFunction extends DrillBaseComputeScalePr
int maxResultIntegerDigits = leftIntegerDigits + rightScale;
- outputPrecision = DecimalUtility.getPrecisionRange(outputScale + maxResultIntegerDigits);
+ outputPrecision = CoreDecimalUtility.getPrecisionRange(outputScale + maxResultIntegerDigits);
// Output precision should be greater or equal to the input precision
outputPrecision = Math.max(outputPrecision, Math.max(leftPrecision, rightPrecision));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java
index e1e587b..1c41197 100644
--- a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.util;
+
public class DecimalScalePrecisionModFunction extends DrillBaseComputeScalePrecision {
public DecimalScalePrecisionModFunction(int leftPrecision, int leftScale, int rightPrecision, int rightScale) {
@@ -31,7 +32,7 @@ public class DecimalScalePrecisionModFunction extends DrillBaseComputeScalePreci
outputScale = Math.max(leftScale, rightScale);
int leftIntegerDigits = leftPrecision - leftScale;
- outputPrecision = DecimalUtility.getPrecisionRange(outputScale + leftIntegerDigits);
+ outputPrecision = CoreDecimalUtility.getPrecisionRange(outputScale + leftIntegerDigits);
if (outputScale + leftIntegerDigits > outputPrecision) {
outputScale = outputPrecision - leftIntegerDigits;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
deleted file mode 100644
index 85ba918..0000000
--- a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
+++ /dev/null
@@ -1,735 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.util;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-import org.apache.drill.common.types.TypeProtos;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.RoundingMode;
-import java.util.Arrays;
-
-public class DecimalUtility {
-
- public final static int MAX_DIGITS = 9;
- public final static int DIGITS_BASE = 1000000000;
- public final static int DIGITS_MAX = 999999999;
- public final static int integerSize = (Integer.SIZE/8);
-
- public final static String[] decimalToString = {"",
- "0",
- "00",
- "000",
- "0000",
- "00000",
- "000000",
- "0000000",
- "00000000",
- "000000000"};
-
- public final static long[] scale_long_constants = {
- 1,
- 10,
- 100,
- 1000,
- 10000,
- 100000,
- 1000000,
- 10000000,
- 100000000,
- 1000000000,
- 10000000000l,
- 100000000000l,
- 1000000000000l,
- 10000000000000l,
- 100000000000000l,
- 1000000000000000l,
- 10000000000000000l,
- 100000000000000000l,
- 1000000000000000000l};
-
- /*
- * Simple function that returns the static precomputed
- * power of ten, instead of using Math.pow
- */
- public static long getPowerOfTen(int power) {
- assert power >= 0 && power < scale_long_constants.length;
- return scale_long_constants[(power)];
- }
-
- /*
- * Math.pow returns a double and while multiplying with large digits
- * in the decimal data type we encounter noise. So instead of multiplying
- * with Math.pow we use the static constants to perform the multiplication
- */
- public static long adjustScaleMultiply(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input * scale_long_constants[index];
- } else {
- return input / scale_long_constants[index];
- }
- }
-
- public static long adjustScaleDivide(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input / scale_long_constants[index];
- } else {
- return input * scale_long_constants[index];
- }
- }
-
- /* Given the number of actual digits this function returns the
- * number of indexes it will occupy in the array of integers
- * which are stored in base 1 billion
- */
- public static int roundUp(int ndigits) {
- return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
- }
-
- /* Returns a string representation of the given integer
- * If the length of the given integer is less than the
- * passed length, this function will prepend zeroes to the string
- */
- public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
- String value = ((Integer) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
- str.append(decimalToString[desiredLength - length]);
- str.append(value);
-
- return str;
- }
-
- public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
- String value = ((Long) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
-
- // Desired length can be > MAX_DIGITS
- int zeroesLength = desiredLength - length;
- while (zeroesLength > MAX_DIGITS) {
- str.append(decimalToString[MAX_DIGITS]);
- zeroesLength -= MAX_DIGITS;
- }
- str.append(decimalToString[zeroesLength]);
- str.append(value);
-
- return str;
- }
-
- public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
- return getBigDecimalFromByteBuf(data, startIndex, nDecimalDigits, scale, false);
- }
-
- public static BigDecimal getBigDecimalFromSparse(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
- return getBigDecimalFromByteBuf(data, startIndex, nDecimalDigits, scale, true);
- }
-
-
- /* Create a BigDecimal object using the data in the ByteBuf.
- * This function assumes that data is provided in a non-dense format
- * It works on both sparse and intermediate representations.
- */
- public static BigDecimal getBigDecimalFromByteBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) {
-
- // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
- int actualDigits;
-
- // Initialize the BigDecimal, first digit in the ByteBuf has the sign so mask it out
- BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
-
- BigInteger base = BigInteger.valueOf(DIGITS_BASE);
-
- for (int i = 1; i < nDecimalDigits; i++) {
-
- BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize)));
- decimalDigits = decimalDigits.multiply(base);
- decimalDigits = decimalDigits.add(temp);
- }
-
- // Truncate any additional padding we might have added
- if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
- BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
- decimalDigits = decimalDigits.divide(truncate);
- }
-
- // set the sign
- if ((data.getInt(startIndex) & 0x80000000) != 0) {
- decimalDigits = decimalDigits.negate();
- }
-
- BigDecimal decimal = new BigDecimal(decimalDigits, scale);
-
- return decimal;
- }
-
- /* This function returns a BigDecimal object from the dense decimal representation.
- * First step is to convert the dense representation into an intermediate representation
- * and then invoke getBigDecimalFromByteBuf() to get the BigDecimal object
- */
- public static BigDecimal getBigDecimalFromDense(ByteBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
-
- /* This method converts the dense representation to
- * an intermediate representation. The intermediate
- * representation has one more integer than the dense
- * representation.
- */
- byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)];
-
- // Start storing from the least significant byte of the first integer
- int intermediateIndex = 3;
-
- int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
- int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
- int maskIndex;
- int shiftOrder;
- byte shiftBits;
-
- // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
- if (maxPrecision == 38) {
- maskIndex = 0;
- shiftOrder = 6;
- shiftBits = 0x00;
- intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
- } else if (maxPrecision == 28) {
- maskIndex = 1;
- shiftOrder = 4;
- shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
- intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
- } else {
- throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
- }
-
- int inputIndex = 1;
- boolean sign = false;
-
- if ((data.getByte(startIndex) & 0x80) != 0) {
- sign = true;
- }
-
- while (inputIndex < width) {
-
- intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
-
- shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
-
- inputIndex++;
- intermediateIndex++;
-
- if (((inputIndex - 1) % integerSize) == 0) {
- shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
- maskIndex++;
- shiftOrder -= 2;
- }
-
- }
- /* copy the last byte */
- intermediateBytes[intermediateIndex] = shiftBits;
-
- if (sign == true) {
- intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
- }
-
- ByteBuf intermediateData = Unpooled.wrappedBuffer(intermediateBytes);
-
- return getBigDecimalFromIntermediate(intermediateData, 0, nDecimalDigits + 1, scale);
- }
-
- /*
- * Function converts the BigDecimal and stores it in out internal sparse representation
- */
- public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision, int nDecimalDigits) {
-
- // Initialize the buffer
- for (int i = 0; i < nDecimalDigits; i++) {
- data.setInt(startIndex + (i * integerSize), 0);
- }
-
- boolean sign = false;
-
- if (input.signum() == -1) {
- // negative input
- sign = true;
- input = input.abs();
- }
-
- // Truncate the input as per the scale provided
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- // Separate out the integer part
- BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
-
- int destIndex = nDecimalDigits - roundUp(scale) - 1;
-
- // we use base 1 billion integer digits for out integernal representation
- BigDecimal base = new BigDecimal(DIGITS_BASE);
-
- while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
- // store the modulo as the integer value
- data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue());
- destIndex--;
- // Divide by base 1 billion
- integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
- }
-
- /* Sparse representation contains padding of additional zeroes
- * so each digit contains MAX_DIGITS for ease of arithmetic
- */
- int actualDigits;
- if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
- // Pad additional zeroes
- scale = scale + (MAX_DIGITS - actualDigits);
- input = input.setScale(scale, BigDecimal.ROUND_DOWN);
- }
-
- //separate out the fractional part
- BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
-
- destIndex = nDecimalDigits - 1;
-
- while (scale > 0) {
- // Get next set of MAX_DIGITS (9) store it in the ByteBuf
- fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
- BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
-
- data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue()));
- destIndex--;
-
- fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
- scale -= MAX_DIGITS;
- }
-
- // Set the negative sign
- if (sign == true) {
- data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
- }
-
- }
- public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate/ or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().intValue());
- }
-
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().longValue());
- }
-
- public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
-
- public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
-
-
- public static int compareDenseBytes(ByteBuf left, int leftStart, boolean leftSign, ByteBuf right, int rightStart, boolean rightSign, int width) {
-
- int invert = 1;
-
- /* If signs are different then simply look at the
- * sign of the two inputs and determine which is greater
- */
- if (leftSign != rightSign) {
-
- return((leftSign == true) ? -1 : 1);
- } else if(leftSign == true) {
- /* Both inputs are negative, at the end we will
- * have to invert the comparison
- */
- invert = -1;
- }
-
- int cmp = 0;
-
- for (int i = 0; i < width; i++) {
- byte leftByte = left.getByte(leftStart + i);
- byte rightByte = right.getByte(rightStart + i);
- // Unsigned byte comparison
- if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
- cmp = 1;
- break;
- } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
- cmp = -1;
- break;
- }
- }
- cmp *= invert; // invert the comparison if both were negative values
-
- return cmp;
- }
-
- public static int getIntegerFromSparseBuffer(ByteBuf buffer, int start, int index) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- value = (value & 0x7FFFFFFF);
- }
- return value;
- }
-
- public static void setInteger(ByteBuf buffer, int start, int index, int value) {
- buffer.setInt(start + (index * 4), value);
- }
-
- public static int compareSparseBytes(ByteBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ByteBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
-
- int invert = 1;
-
- if (absCompare == false) {
- if (leftSign != rightSign) {
- return (leftSign == true) ? -1 : 1;
- }
-
- // Both values are negative invert the outcome of the comparison
- if (leftSign == true) {
- invert = -1;
- }
- }
-
- int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
- return cmp * invert;
- }
- public static int compareSparseBytesInner(ByteBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ByteBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
- /* compute the number of integer digits in each decimal */
- int leftInt = leftPrecision - leftScale;
- int rightInt = rightPrecision - rightScale;
-
- /* compute the number of indexes required for storing integer digits */
- int leftIntRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(leftInt);
- int rightIntRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(rightInt);
-
- /* compute number of indexes required for storing scale */
- int leftScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(leftScale);
- int rightScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(rightScale);
-
- /* compute index of the most significant integer digits */
- int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
- int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
-
- /* Discard the zeroes in the integer part */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- leftIntRoundedUp--;
- leftIndex1++;
- }
-
- /* If we reached the stop index then the number of integers is zero */
- if (leftIndex1 == leftStopIndex) {
- leftIntRoundedUp = 0;
- }
-
- while (rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- rightIntRoundedUp--;
- rightIndex1++;
- }
-
- if (rightIndex1 == rightStopIndex) {
- rightIntRoundedUp = 0;
- }
-
- /* We have the accurate number of non-zero integer digits,
- * if the number of integer digits are different then we can determine
- * which decimal is larger and needn't go down to comparing individual values
- */
- if (leftIntRoundedUp > rightIntRoundedUp) {
- return 1;
- }
- else if (rightIntRoundedUp > leftIntRoundedUp) {
- return -1;
- }
-
- /* The number of integer digits are the same, set the each index
- * to the first non-zero integer and compare each digit
- */
- leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* The integer part of both the decimal's are equal, now compare
- * each individual fractional part. Set the index to be at the
- * beginning of the fractional part
- */
- leftIndex1 = leftStopIndex;
- rightIndex1 = rightStopIndex;
-
- /* Stop indexes will be the end of the array */
- leftStopIndex = nDecimalDigits;
- rightStopIndex = nDecimalDigits;
-
- /* compare the two fractional parts of the decimal */
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* Till now the fractional part of the decimals are equal, check
- * if one of the decimal has fractional part that is remaining
- * and is non-zero
- */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- return 1;
- }
- leftIndex1++;
- }
-
- while(rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- return -1;
- }
- rightIndex1++;
- }
-
- /* Both decimal values are equal */
- return 0;
- }
-
- public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
- byte[] value = Arrays.copyOfRange(bytes, start, start + length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
-
- public static BigDecimal getBigDecimalFromByteBuf(ByteBuf bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.getBytes(start, value, 0, length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- public static void roundDecimal(ByteBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
- int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(desiredScale);
- int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(currentScale);
-
- if (desiredScale < currentScale) {
-
- boolean roundUp = false;
-
- //Extract the first digit to be truncated to check if we need to round up
- int truncatedScaleIndex = desiredScale + 1;
- if (truncatedScaleIndex <= currentScale) {
- int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
- extractDigitIndex += org.apache.drill.common.util.DecimalUtility.roundUp(truncatedScaleIndex);
- int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
- int temp = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
- if (temp != 0) {
- extractDigit = extractDigit / (int) (Math.pow(10, temp));
- }
- if ((extractDigit % 10) > 4) {
- roundUp = true;
- }
- }
-
- // Get the source index beyond which we will truncate
- int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
- int srcIndex = srcIntIndex + newScaleRoundedUp;
-
- // Truncate the remaining fractional part, move the integer part
- int destIndex = nDecimalDigits - 1;
- if (srcIndex != destIndex) {
- while (srcIndex >= 0) {
- setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
- }
-
- // Set the remaining portion of the decimal to be zeroes
- while (destIndex >= 0) {
- setInteger(result, start, destIndex--, 0);
- }
- srcIndex = nDecimalDigits - 1;
- }
-
- // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
- int truncateFactor = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
- if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
- truncateFactor = (int) Math.pow(10, truncateFactor);
- int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
- fractionalDigits /= truncateFactor;
- setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
- }
-
- // Finally round up the digit if needed
- if (roundUp == true) {
- srcIndex = nDecimalDigits - 1;
- int carry;
- if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
- carry = truncateFactor;
- } else {
- carry = 1;
- }
-
- while (srcIndex >= 0) {
- int value = getIntegerFromSparseBuffer(result, start, srcIndex);
- value += carry;
-
- if (value >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- setInteger(result, start, srcIndex--, value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- carry = value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
- } else {
- setInteger(result, start, srcIndex--, value);
- carry = 0;
- break;
- }
- }
- }
- } else if (desiredScale > currentScale) {
- // Add fractional digits to the decimal
-
- // Check if we need to shift the decimal digits to the left
- if (newScaleRoundedUp > origScaleRoundedUp) {
- int srcIndex = 0;
- int destIndex = newScaleRoundedUp - origScaleRoundedUp;
-
- // Check while extending scale, we are not overwriting integer part
- while (srcIndex < destIndex) {
- if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
- }
- }
-
- srcIndex = 0;
- while (destIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
- }
-
- // Clear the remaining part
- while (srcIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, 0);
- }
- }
- }
- }
-
- /*
- * Function returns the Minor decimal type given the precision
- */
- public static TypeProtos.MinorType getDecimalDataType(int precision) {
- if (precision <= 9) {
- return TypeProtos.MinorType.DECIMAL9;
- } else if (precision <= 18) {
- return TypeProtos.MinorType.DECIMAL18;
- } else if (precision <= 28) {
- return TypeProtos.MinorType.DECIMAL28SPARSE;
- } else {
- return TypeProtos.MinorType.DECIMAL38SPARSE;
- }
- }
-
- public static int getMaxPrecision(TypeProtos.MinorType decimalType) {
- if (decimalType == TypeProtos.MinorType.DECIMAL9) {
- return 9;
- } else if (decimalType == TypeProtos.MinorType.DECIMAL18) {
- return 18;
- } else if (decimalType == TypeProtos.MinorType.DECIMAL28SPARSE) {
- return 28;
- } else if (decimalType == TypeProtos.MinorType.DECIMAL38SPARSE) {
- return 38;
- }
- return 0;
- }
-
-
- /*
- * Given a precision it provides the max precision of that decimal data type;
- * For eg: given the precision 12, we would use DECIMAL18 to store the data
- * which has a max precision range of 18 digits
- */
- public static int getPrecisionRange(int precision) {
- return getMaxPrecision(getDecimalDataType(precision));
- }
-
- public static int getFirstFractionalDigit(int decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- int temp = (int) adjustScaleDivide(decimal, scale - 1);
- return Math.abs(temp % 10);
- }
-
- public static int getFirstFractionalDigit(long decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- long temp = adjustScaleDivide(decimal, scale - 1);
- return (int) (Math.abs(temp % 10));
- }
-
- public static int getFirstFractionalDigit(ByteBuf data, int scale, int start, int nDecimalDigits) {
- if (scale == 0) {
- return 0;
- }
-
- int index = nDecimalDigits - roundUp(scale);
- return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1));
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
index 2267ad5..576fa47 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
@@ -24,7 +24,7 @@
package org.apache.drill.exec.expr.fn.impl.hive;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.*;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 903f953..d1f9754 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -21,10 +21,16 @@
<name>exec/Java Execution Engine</name>
<dependencies>
+<!-- <dependency> -->
+<!-- <groupId>org.ow2.asm</groupId> -->
+<!-- <artifactId>asm-util</artifactId> -->
+<!-- <version>4.1</version> -->
+<!-- </dependency> -->
<dependency>
<groupId>org.ow2.asm</groupId>
- <artifactId>asm-util</artifactId>
+ <artifactId>asm-debug-all</artifactId>
<version>4.1</version>
+<!-- <scope>test</scope> -->
</dependency>
<dependency>
<groupId>pentaho</groupId>
@@ -38,11 +44,11 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm-commons</artifactId>
- <version>4.1</version>
- </dependency>
+<!-- <dependency> -->
+<!-- <groupId>org.ow2.asm</groupId> -->
+<!-- <artifactId>asm-commons</artifactId> -->
+<!-- <version>4.1</version> -->
+<!-- </dependency> -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index 2b76514..a4f5fb1 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -78,7 +78,7 @@
{from: "IntervalDay", to: "VarChar", major: "IntervalDayVarChar", bufferLength: "43"},
{from: "IntervalYear", to: "Interval", major: "IntervalSimpleToComplex", months: "in.value", days: "0", millis: "0"},
- {from: "IntervalDay", to: "Interval", major: "IntervalSimpleToComplex", months: "0", days: "in.days", millis: "in.milliSeconds"},
+ {from: "IntervalDay", to: "Interval", major: "IntervalSimpleToComplex", months: "0", days: "in.days", millis: "in.milliseconds"},
{from: "Interval", to: "IntervalYear", major: "IntervalComplexToSimple"},
{from: "Interval", to: "IntervalDay", major: "IntervalComplexToSimple"}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
index d3b3ac4..813f43a 100644
--- a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
@@ -26,9 +26,10 @@
width: 1,
javaType: "byte",
boxedType: "Byte",
+ fields: [{name: "value", type: "byte"}],
minor: [
- { class: "TinyInt", valueHolder: "IntHolder"},
- { class: "UInt1", valueHolder: "UInt1Holder"}
+ { class: "TinyInt", valueHolder: "IntHolder" },
+ { class: "UInt1", valueHolder: "UInt1Holder" }
]
},
{
@@ -36,6 +37,7 @@
width: 2,
javaType: "char",
boxedType: "Character",
+ fields: [{name: "value", type: "char"}],
minor: [
{ class: "UInt2", valueHolder: "UInt2Holder"}
]
@@ -44,6 +46,7 @@
width: 2,
javaType: "short",
boxedType: "Short",
+ fields: [{name: "value", type: "short"}],
minor: [
{ class: "SmallInt", valueHolder: "Int2Holder"},
]
@@ -53,13 +56,14 @@
width: 4,
javaType: "int",
boxedType: "Integer",
+ fields: [{name: "value", type: "int"}],
minor: [
- { class: "Int", valueHolder: "IntHolder" },
+ { class: "Int", valueHolder: "IntHolder"},
{ class: "UInt4", valueHolder: "UInt4Holder" },
- { class: "Float4", javaType: "float" , boxedType: "Float" },
+ { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
{ class: "Time", javaType: "int", friendlyType: "DateTime" },
{ class: "IntervalYear", javaType: "int", friendlyType: "Period" }
- { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal" },
+ { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
]
},
{
@@ -67,13 +71,14 @@
width: 8,
javaType: "long",
boxedType: "Long",
+ fields: [{name: "value", type: "long"}],
minor: [
- { class: "BigInt" },
+ { class: "BigInt"},
{ class: "UInt8" },
- { class: "Float8", javaType: "double" , boxedType: "Double" },
+ { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
{ class: "Date", javaType: "long", friendlyType: "DateTime" },
{ class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
- { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal" },
+ { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
<#--
{ class: "Money", maxPrecisionDigits: 2, scale: 1, },
-->
@@ -82,68 +87,69 @@
{
major: "Fixed",
width: 12,
- javaType: "ByteBuf",
- boxedType: "ByteBuf",
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
minor: [
- { class: "TimeStampTZ", milliSecondsSize: 8, friendlyType: "DateTime" },
- { class: "IntervalDay", milliSecondsOffset: 4, friendlyType: "Period" }
+ { class: "TimeStampTZ", millisecondsSize: 8, friendlyType: "DateTime", fields: [ {name: "value", type:"long"}, {name: "index", type: "int"}] },
+ { class: "IntervalDay", millisecondsOffset: 4, friendlyType: "Period", fields: [ {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
]
},
{
major: "Fixed",
width: 16,
- javaType: "ByteBuf"
- boxedType: "ByteBuf",
+ javaType: "DrillBuf"
+ boxedType: "DrillBuf",
minor: [
- { class: "Interval", daysOffset: 4, milliSecondsOffset: 8, friendlyType: "Period" }
+ { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
]
},
{
major: "Fixed",
width: 12,
- javaType: "ByteBuf",
- boxedType: "ByteBuf",
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
minor: [
<#--
{ class: "TimeTZ" },
{ class: "Interval" }
-->
- { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal" }
+ { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
]
},
{
major: "Fixed",
width: 16,
- javaType: "ByteBuf",
- boxedType: "ByteBuf",
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
minor: [
- { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal" }
+ { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
]
},
{
major: "Fixed",
width: 24,
- javaType: "ByteBuf",
- boxedType: "ByteBuf",
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
minor: [
- { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal" }
+ { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
]
},
{
major: "Fixed",
width: 20,
- javaType: "ByteBuf",
- boxedType: "ByteBuf",
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
minor: [
- { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal" }
+ { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
]
},
{
major: "VarLen",
width: 4,
javaType: "int",
- boxedType: "ByteBuf",
+ boxedType: "DrillBuf",
+ fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "DrillBuf"}],
minor: [
{ class: "VarBinary" , friendlyType: "byte[]" },
{ class: "VarChar" , friendlyType: "Text" }
@@ -154,9 +160,9 @@
major: "Bit",
width: 1,
javaType: "int",
- boxedType: "ByteBuf",
+ boxedType: "Integer",
minor: [
- { class: "Bit" , friendlyType: "Boolean" }
+ { class: "Bit" , friendlyType: "Boolean", fields: [{name: "value", type: "int"}] }
]
}
]
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 67c6bff..371d8d0 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -45,6 +45,8 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import com.sun.codemodel.JType;
import com.sun.codemodel.JCodeModel;
+import javax.inject.Inject;
+
import java.util.Arrays;
import java.util.Random;
import java.util.List;
@@ -65,7 +67,7 @@ import org.joda.time.Period;
import org.apache.hadoop.io.Text;
import org.apache.drill.exec.vector.accessor.sql.TimePrintMillis;
-
+import javax.inject.Inject;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java b/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java
index e2fd9d5..c261e29 100644
--- a/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java
+++ b/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java
@@ -28,7 +28,10 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -51,11 +54,11 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]);
+ buffer = buffer.reallocIfNeeded((int) len.value);
}
public void eval() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctions.java b/exec/java-exec/src/main/codegen/templates/CastFunctions.java
index 4c750dd..41c1557 100644
--- a/exec/java-exec/src/main/codegen/templates/CastFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/CastFunctions.java
@@ -34,6 +34,8 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.record.RecordBatch;
@SuppressWarnings("unused")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
index aa216cd..d8ce44a 100644
--- a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
+++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
@@ -41,6 +41,8 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java
index 8594d5a..0096be8 100644
--- a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java
+++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java
@@ -38,6 +38,8 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java
index 319ab6b..455ab13 100644
--- a/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java
+++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java
@@ -29,6 +29,7 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -38,6 +39,8 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
@@ -45,15 +48,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- //TODO: max bufferLength should = parameter.len
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]);
}
public void eval() {
+ buffer = buffer.reallocIfNeeded((int) len.value);
String istr = (new ${type.javaType}(in.value)).toString();
out.buffer = buffer;
out.start = 0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java b/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java
index 19c218f..e85b724 100644
--- a/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java
+++ b/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java
@@ -26,6 +26,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
@@ -54,7 +56,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
public void eval() {
out.months = ${type.months};
out.days = ${type.days};
- out.milliSeconds = ${type.millis};
+ out.milliseconds = ${type.millis};
}
}
@@ -96,7 +98,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
out.value = in.months;
<#elseif type.to == "IntervalDay">
out.days = in.days;
- out.milliSeconds = in.milliSeconds;
+ out.milliseconds = in.milliseconds;
</#if>
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java b/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java
index 4c88fcc..829fe47 100644
--- a/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java
+++ b/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java
@@ -27,7 +27,10 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -49,11 +52,11 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]);
+ buffer.reallocIfNeeded(${type.bufferLength});
}
public void eval() {
@@ -61,7 +64,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
int years = (in.months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
int months = (in.months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
- long millis = in.milliSeconds;
+ long millis = in.milliseconds;
long hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
@@ -99,6 +102,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
import io.netty.buffer.ByteBuf;
@@ -122,11 +126,11 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]);
+ buffer = buffer.reallocIfNeeded((int) len.value);
}
public void eval() {
@@ -154,6 +158,9 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+
+
+
import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.expr.DrillSimpleFunc;
@@ -169,6 +176,8 @@ import org.joda.time.MutableDateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.DateMidnight;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL,
@@ -177,16 +186,15 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]);
+ buffer = buffer.reallocIfNeeded((int) len.value);
}
public void eval() {
-
- long millis = in.milliSeconds;
+ long millis = in.milliseconds;
long hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
index e2e1143..cf3ae90 100644
--- a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
+++ b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
@@ -40,6 +40,8 @@ import org.joda.time.MutableDateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.DateMidnight;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(names = {"cast${type.to?upper_case}", "${type.alias}"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL,
@@ -48,19 +50,20 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Output ${type.to}Holder out;
-
+
public void setup(RecordBatch incoming) {
}
public void eval() {
+ <#if type.to != "Date">
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
-
+ </#if>
+
<#if type.to == "Date">
- org.joda.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getDateTimeFormatter();
- out.value = (org.joda.time.DateMidnight.parse(input, f).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis();
+ out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getDate(in.buffer.memoryAddress(), in.start, in.end);
<#elseif type.to == "TimeStamp">
org.joda.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getDateTimeFormatter();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
index 135abdd..eaebf02 100644
--- a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
+++ b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
@@ -40,6 +40,8 @@ import org.joda.time.MutableDateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.DateMidnight;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
@@ -65,7 +67,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
out.days = period.getDays();
- out.milliSeconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
+ out.milliseconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
(period.getMinutes() * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) +
(period.getSeconds() * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) +
(period.getMillis());
@@ -73,7 +75,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
<#elseif type.to == "IntervalDay">
out.days = period.getDays();
- out.milliSeconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
+ out.milliseconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) +
(period.getMinutes() * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) +
(period.getSeconds() * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) +
(period.getMillis());
[27/47] git commit: Set the BitVector memory to zero on allocation.
Posted by ja...@apache.org.
Set the BitVector memory to zero on allocation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f1a46e68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f1a46e68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f1a46e68
Branch: refs/heads/master
Commit: f1a46e687ba5572b1cc0c830a4ea787876d5ad7c
Parents: 45d6537
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Mon Aug 25 21:10:36 2014 +0530
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon Aug 25 21:10:36 2014 +0530
----------------------------------------------------------------------
.../src/main/codegen/templates/NullableValueVectors.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1a46e68/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index ec99e79..cb52841 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -114,6 +114,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public boolean allocateNewSafe() {
if(!values.allocateNewSafe()) return false;
if(!bits.allocateNewSafe()) return false;
+ bits.zeroVector();
mutator.reset();
accessor.reset();
return true;
@@ -123,6 +124,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void allocateNew(int totalBytes, int valueCount) {
values.allocateNew(totalBytes, valueCount);
bits.allocateNew(valueCount);
+ bits.zeroVector();
mutator.reset();
accessor.reset();
}
@@ -171,6 +173,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void allocateNew() {
values.allocateNew();
bits.allocateNew();
+ bits.zeroVector();
mutator.reset();
accessor.reset();
}
@@ -180,6 +183,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public boolean allocateNewSafe() {
if(!values.allocateNewSafe()) return false;
if(!bits.allocateNewSafe()) return false;
+ bits.zeroVector();
mutator.reset();
accessor.reset();
return true;
@@ -189,6 +193,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void allocateNew(int valueCount) {
values.allocateNew(valueCount);
bits.allocateNew(valueCount);
+ bits.zeroVector();
mutator.reset();
accessor.reset();
}
@@ -198,6 +203,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
*/
public void zeroVector() {
this.values.zeroVector();
+ this.bits.zeroVector();
}
@Override
[14/47] git commit: DRILL-1312: Use correct index to get partition
vectors in ScanBatch
Posted by ja...@apache.org.
DRILL-1312: Use correct index to get partition vectors in ScanBatch
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c3e97fdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c3e97fdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c3e97fdf
Branch: refs/heads/master
Commit: c3e97fdfab67ff5d81d478e26534c3937fada07c
Parents: a6d74f5
Author: Mehant Baid <me...@gmail.com>
Authored: Thu Aug 21 17:31:33 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 12:05:26 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/physical/impl/ScanBatch.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3e97fdf/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 9472627..e999c9a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -224,9 +224,10 @@ public class ScanBatch implements RecordBatch {
}
private void populatePartitionVectors() {
- for (int i : selectedPartitionColumns) {
+ for (int index = 0; index < selectedPartitionColumns.size(); index++) {
+ int i = selectedPartitionColumns.get(index);
+ NullableVarCharVector v = (NullableVarCharVector) partitionVectors.get(index);
if (partitionValues.length > i) {
- NullableVarCharVector v = (NullableVarCharVector) partitionVectors.get(i);
String val = partitionValues[i];
AllocationHelper.allocate(v, recordCount, val.length());
NullableVarCharHolder h = new NullableVarCharHolder();
@@ -241,7 +242,6 @@ public class ScanBatch implements RecordBatch {
}
v.getMutator().setValueCount(recordCount);
} else {
- NullableVarCharVector v = (NullableVarCharVector) partitionVectors.get(i);
AllocationHelper.allocate(v, recordCount, 0);
v.getMutator().setValueCount(recordCount);
}
[26/47] git commit: Fixed the index check in DrillBuf.
Posted by ja...@apache.org.
Fixed the index check in DrillBuf.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/45d65377
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/45d65377
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/45d65377
Branch: refs/heads/master
Commit: 45d65377d7de2cd1e58bb4e1b0650bb2c53eeff1
Parents: 5c5cef0
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Mon Aug 25 21:09:16 2014 +0530
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon Aug 25 21:09:16 2014 +0530
----------------------------------------------------------------------
exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/45d65377/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java b/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
index fb70f89..6ad5da8 100644
--- a/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
@@ -161,7 +161,7 @@ public final class DrillBuf extends AbstractByteBuf {
private void chk(int index, int width) {
if (AssertionUtil.isAssertionsEnabled()) {
- checkIndexD(offset+index, width);
+ checkIndexD(index, width);
}
}
[13/47] git commit: DRILL-1132: WriterPrel should not care about
collation trait of the child.
Posted by ja...@apache.org.
DRILL-1132: WriterPrel should not care about collation trait of the child.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a6d74f5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a6d74f5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a6d74f5d
Branch: refs/heads/master
Commit: a6d74f5d1ea03314119a1b404a515251fd246a3f
Parents: de05070
Author: Aman Sinha <as...@maprtech.com>
Authored: Fri Aug 22 12:01:39 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 12:04:29 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/planner/physical/WriterPrule.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a6d74f5d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrule.java
index 15d94fb..f3057e6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrule.java
@@ -21,13 +21,10 @@ import org.apache.drill.exec.planner.common.DrillWriterRelBase;
import org.apache.drill.exec.planner.logical.DrillRel;
import org.apache.drill.exec.planner.logical.DrillWriterRel;
import org.apache.drill.exec.planner.logical.RelOptHelper;
-import org.eigenbase.rel.RelCollation;
-import org.eigenbase.rel.RelCollationTraitDef;
import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.RelOptRule;
import org.eigenbase.relopt.RelOptRuleCall;
import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.relopt.volcano.RelSubset;
public class WriterPrule extends Prule{
public static final RelOptRule INSTANCE = new WriterPrule();
@@ -62,10 +59,14 @@ public class WriterPrule extends Prule{
@Override
public RelNode convertChild(DrillWriterRelBase writer, RelNode rel) throws RuntimeException {
DrillDistributionTrait childDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
- RelCollation childCollation = rel.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE);
+ // Create the Writer with the child's distribution because the degree of parallelism for the writer
+ // should correspond to the number of child minor fragments. The Writer itself is not concerned with
+ // the collation of the child. Note that the Writer's output RowType consists of
+ // {fragment_id varchar(255), number_of_records_written bigint} which are very different from the
+ // child's output RowType.
return new WriterPrel(writer.getCluster(),
- writer.getTraitSet().plus(childDist).plus(childCollation).plus(Prel.DRILL_PHYSICAL),
+ writer.getTraitSet().plus(childDist).plus(Prel.DRILL_PHYSICAL),
rel, writer.getCreateTableEntry());
}
[33/47] git commit: DRILL-595, DRILL-811,
DRILL-1029: Support select * in view, subquery,
table expression in with clause. Most fixes in Optiq.
Posted by ja...@apache.org.
DRILL-595, DRILL-811, DRILL-1029: Support select * in view, subquery, table expression in with clause. Most fixes in Optiq.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4f59d4f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4f59d4f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4f59d4f0
Branch: refs/heads/master
Commit: 4f59d4f0f835fec935cbdf05a277015cb763b9e4
Parents: c4cc9e4
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Fri Aug 15 17:21:38 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 13:33:47 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/dotdrill/View.java | 10 ++
.../drill/exec/planner/StarColumnHelper.java | 10 +-
.../drill/exec/planner/logical/DrillOptiq.java | 10 ++
.../exec/planner/logical/DrillViewTable.java | 3 +-
.../org/apache/drill/TestExampleQueries.java | 150 +++++++++++++++++++
pom.xml | 2 +-
6 files changed, 182 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f59d4f0/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
index 7965e02..8119c29 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.dotdrill;
import java.util.List;
import com.google.common.collect.ImmutableList;
+import org.apache.drill.exec.planner.StarColumnHelper;
import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl;
import org.apache.drill.exec.planner.types.RelDataTypeHolder;
import org.eigenbase.reltype.RelDataType;
@@ -135,6 +136,15 @@ public class View {
return fields.isEmpty();
}
+ @JsonIgnore
+ public boolean hasStar() {
+ for (FieldType field : fields) {
+ if (StarColumnHelper.isNonPrefixedStarColumn(field.name))
+ return true;
+ }
+ return false;
+ }
+
public String getSql() {
return sql;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f59d4f0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
index 4c04b22..591faf9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java
@@ -45,7 +45,15 @@ public class StarColumnHelper {
public static boolean isPrefixedStarColumn(String fieldName) {
return fieldName.indexOf(PREFIXED_STAR_COLUMN) > 0 ; // the delimiter * starts at none-zero position.
}
-
+
+ public static boolean isNonPrefixedStarColumn(String fieldName) {
+ return fieldName.startsWith("*");
+ }
+
+ public static boolean isStarColumn(String fieldName) {
+ return isPrefixedStarColumn(fieldName) || isNonPrefixedStarColumn(fieldName);
+ }
+
public static String extractStarColumnPrefix(String fieldName) {
assert (isPrefixedStarColumn(fieldName));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f59d4f0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index aa88310..891b889 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -36,6 +36,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.planner.StarColumnHelper;
import org.eigenbase.rel.RelNode;
import org.eigenbase.reltype.RelDataTypeField;
import org.eigenbase.rex.RexCall;
@@ -167,6 +168,15 @@ public class DrillOptiq {
if (call.getOperator() == SqlStdOperatorTable.ITEM) {
SchemaPath left = (SchemaPath) call.getOperands().get(0).accept(this);
+
+ // Convert expr of item[*, 'abc'] into column expression 'abc'
+ String rootSegName = left.getRootSegment().getPath();
+ if (StarColumnHelper.isStarColumn(rootSegName)) {
+ rootSegName = rootSegName.substring(0, rootSegName.indexOf("*"));
+ final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
+ return SchemaPath.getSimplePath(rootSegName + literal.getValue2().toString());
+ }
+
final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
switch(literal.getTypeName()){
case DECIMAL:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f59d4f0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
index e798d4e..68e666a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
@@ -56,7 +56,8 @@ public class DrillViewTable implements TranslatableTable, DrillViewInfoProvider
RelDataType rowType = relOptTable.getRowType();
RelNode rel = context.expandView(rowType, view.getSql(), view.getWorkspaceSchemaPath());
- if (view.isDynamic()){
+ if (view.isDynamic() || view.hasStar()){
+ // if View's field has "*", return rel directly.
return rel;
}else{
// if the View's field list is not "*", try to create a cast.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f59d4f0/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index d9e8b20..0053388 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.rpc.RpcException;
import org.junit.Ignore;
import org.junit.Test;
@@ -321,4 +322,153 @@ public class TestExampleQueries extends BaseTestQuery{
test(query);
}
+ @Test
+ public void testSelStarSubQJson2() throws Exception {
+ test("select v.first_name from (select * from cp.`employee.json`) v limit 2" );
+ }
+
+ // Select * in SubQuery, View or CTE (With clause)
+ @Test // Select * in SubQuery : regular columns appear in select clause, where, group by, order by.
+ public void testSelStarSubQPrefix() throws Exception {
+ test("select t.n_nationkey, t.n_name, t.n_regionkey from (select * from cp.`tpch/nation.parquet`) t where t.n_regionkey > 1 order by t.n_name" );
+
+ test("select n.n_regionkey, count(*) as cnt from ( select * from ( select * from cp.`tpch/nation.parquet`) t where t.n_nationkey < 10 ) n where n.n_nationkey >1 group by n.n_regionkey order by n.n_regionkey ; ");
+
+ test("select t.n_regionkey, count(*) as cnt from (select * from cp.`tpch/nation.parquet`) t where t.n_nationkey > 1 group by t.n_regionkey order by t.n_regionkey;" );
+ }
+
+ @Test // Select * in SubQuery : regular columns appear in select clause, where, group by, order by.
+ public void testSelStarSubQNoPrefix() throws Exception {
+ test("select n_nationkey, n_name, n_regionkey from (select * from cp.`tpch/nation.parquet`) where n_regionkey > 1 order by n_name" );
+
+ test("select n_regionkey, count(*) as cnt from ( select * from ( select * from cp.`tpch/nation.parquet`) where n_nationkey < 10 ) where n_nationkey >1 group by n_regionkey order by n_regionkey ; ");
+
+ test("select n_regionkey, count(*) as cnt from (select * from cp.`tpch/nation.parquet`) t where n_nationkey > 1 group by n_regionkey order by n_regionkey;" );
+ }
+
+ @Test // join two SubQuery, each having select * : regular columns appear in the select , where and on clause, group by, order by.
+ public void testSelStarSubQJoin() throws Exception {
+ // select clause, where.
+ test(" select n.n_nationkey, n.n_name, n.n_regionkey, r.r_name \n" +
+ " from (select * from cp.`tpch/nation.parquet`) n, \n" +
+ " (select * from cp.`tpch/region.parquet`) r \n" +
+ " where n.n_regionkey = r.r_regionkey " );
+
+ // select clause, where, group by, order by
+ test(" select n.n_regionkey, count(*) as cnt \n" +
+ " from (select * from cp.`tpch/nation.parquet`) n \n" +
+ " , (select * from cp.`tpch/region.parquet`) r \n" +
+ " where n.n_regionkey = r.r_regionkey and n.n_nationkey > 10 \n" +
+ " group by n.n_regionkey \n" +
+ " order by n.n_regionkey; " );
+
+ // select clause, where, on, group by, order by.
+ test(" select n.n_regionkey, count(*) as cnt \n" +
+ " from (select * from cp.`tpch/nation.parquet`) n \n" +
+ " join (select * from cp.`tpch/region.parquet`) r \n" +
+ " on n.n_regionkey = r.r_regionkey \n" +
+ " where n.n_nationkey > 10 \n" +
+ " group by n.n_regionkey \n" +
+ " order by n.n_regionkey; " );
+
+ // Outer query use select *. Join condition in where clause.
+ test(" select * \n" +
+ " from (select * from cp.`tpch/nation.parquet`) n \n" +
+ " , (select * from cp.`tpch/region.parquet`) r \n" +
+ " where n.n_regionkey = r.r_regionkey " );
+
+ // Outer query use select *. Join condition in on clause.
+ test(" select * \n" +
+ " from (select * from cp.`tpch/nation.parquet`) n \n" +
+ " join (select * from cp.`tpch/region.parquet`) r \n" +
+ " on n.n_regionkey = r.r_regionkey " );
+ }
+
+ @Test // DRILL-595 : Select * in CTE WithClause : regular columns appear in select clause, where, group by, order by.
+ public void testDRILL_595WithClause() throws Exception {
+ test(" with x as (select * from cp.`region.json`) \n" +
+ " select x.region_id, x.sales_city \n" +
+ " from x where x.region_id > 10 limit 5;");
+
+ test(" with x as (select * from cp.`region.json`) \n" +
+ " select region_id, sales_city \n" +
+ " from x where region_id > 10 limit 5;");
+
+ test(" with x as (select * from cp.`tpch/nation.parquet`) \n" +
+ " select x.n_regionkey, count(*) as cnt \n" +
+ " from x \n" +
+ " where x.n_nationkey > 5 \n" +
+ " group by x.n_regionkey \n" +
+ " order by cnt limit 5; ");
+
+ }
+
+ @Test // DRILL-595 : Join two CTE, each having select * : regular columns appear in the select , where and on clause, group by, order by.
+ public void testDRILL_595WithClauseJoin() throws Exception {
+ test("with n as (select * from cp.`tpch/nation.parquet`), \n " +
+ " r as (select * from cp.`tpch/region.parquet`) \n" +
+ "select n.n_nationkey, n.n_name, n.n_regionkey, r.r_name \n" +
+ "from n, r \n" +
+ "where n.n_regionkey = r.r_regionkey ;" );
+
+ test("with n as (select * from cp.`tpch/nation.parquet`), \n " +
+ " r as (select * from cp.`tpch/region.parquet`) \n" +
+ "select n.n_regionkey, count(*) as cnt \n" +
+ "from n, r \n" +
+ "where n.n_regionkey = r.r_regionkey and n.n_nationkey > 5 \n" +
+ "group by n.n_regionkey \n" +
+ "order by cnt;" );
+ }
+
+ @Test // DRILL-811
+ public void testDRILL_811View() throws Exception {
+ test("use dfs.tmp");
+ test("create view nation_view as select * from cp.`tpch/nation.parquet`;");
+
+ test("select n.n_nationkey, n.n_name, n.n_regionkey from nation_view n where n.n_nationkey > 8 order by n.n_regionkey");
+
+ test("select n.n_regionkey, count(*) as cnt from nation_view n where n.n_nationkey > 8 group by n.n_regionkey order by n.n_regionkey");
+
+ test("drop view nation_view ");
+ }
+
+ @Test // DRILL-811
+ public void testDRILL_811ViewJoin() throws Exception {
+ test("use dfs.tmp");
+ test("create view nation_view as select * from cp.`tpch/nation.parquet`;");
+ test("create view region_view as select * from cp.`tpch/region.parquet`;");
+
+ test("select n.n_nationkey, n.n_regionkey, r.r_name from region_view r , nation_view n where r.r_regionkey = n.n_regionkey ");
+
+ test("select n.n_regionkey, count(*) as cnt from region_view r , nation_view n where r.r_regionkey = n.n_regionkey and n.n_nationkey > 8 group by n.n_regionkey order by n.n_regionkey");
+
+ test("select n.n_regionkey, count(*) as cnt from region_view r join nation_view n on r.r_regionkey = n.n_regionkey and n.n_nationkey > 8 group by n.n_regionkey order by n.n_regionkey");
+
+ test("drop view region_view ");
+ test("drop view nation_view ");
+ }
+
+ @Test // DRILL-811
+ public void testDRILL_811Json() throws Exception {
+ test("use dfs.tmp");
+ test("create view region_view as select * from cp.`region.json`;");
+ test("select sales_city, sales_region from region_view where region_id > 50 order by sales_country; ");
+ test("drop view region_view ");
+ }
+
+ @Test(expected = RpcException.class) // Should get "At line 1, column 8: Column 'n_nationkey' is ambiguous"
+ public void testSelStarAmbiguousJoin() throws Exception {
+ test("select x.n_nationkey, x.n_name, x.n_regionkey, x.r_name from (select * from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r where n.n_regionkey = r.r_regionkey) x " ) ;
+ }
+
+ @Test // select star for a SchemaTable.
+ public void testSelStarSubQSchemaTable() throws Exception {
+ test("select name, kind, type from (select * from sys.options);");
+ }
+
+ @Test // Join a select star of SchemaTable, with a select star of Schema-less table.
+ public void testSelStarJoinSchemaWithSchemaLess() throws Exception {
+ test("select t1.name, t1.kind, t2.n_nationkey from (select * from sys.options) t1 join (select * from cp.`tpch/nation.parquet`) t2 on t1.name = t2.n_name;");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f59d4f0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f04d5b2..de02c92 100644
--- a/pom.xml
+++ b/pom.xml
@@ -664,7 +664,7 @@
<dependency>
<groupId>net.hydromatic</groupId>
<artifactId>optiq-core</artifactId>
- <version>0.9-SNAPSHOT</version>
+ <version>0.9-drill-r2</version>
<exclusions>
<exclusion>
<groupId>org.jgrapht</groupId>
[23/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
index aac45e0..7750c62 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
@@ -28,6 +28,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -36,7 +38,10 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -45,16 +50,15 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
<#if type.to.startsWith("Decimal28") || type.to.startsWith("Decimal38")>
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
</#if>
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = (${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
@@ -64,7 +68,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// Re initialize the buffer everytime
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
out.scale = (int) scale.value;
@@ -72,7 +76,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
out.buffer = buffer;
out.start = 0;
- out.setSign((in.value < 0));
+ out.setSign((in.value < 0), out.start, out.buffer);
/* Since we will be dividing the decimal value with base 1 billion
* we don't want negative results if the decimal is negative.
@@ -84,8 +88,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// store the decimal value as sequence of integers of base 1 billion.
while (value > 0) {
- out.setInteger(index, (int) (value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
- value = value/org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ out.setInteger(index, (int) (value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), out.start, out.buffer);
+ value = value/org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
index--;
}
@@ -98,7 +102,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
int shiftOrder = 2;
// Start shifting bits just after the first integer
- int byteIndex = in.WIDTH - (org.apache.drill.common.util.DecimalUtility.integerSize + 1);
+ int byteIndex = in.WIDTH - (org.apache.drill.exec.util.DecimalUtility.integerSize + 1);
while (byteIndex >= 0) {
@@ -129,6 +133,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -137,7 +143,9 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -145,15 +153,14 @@ import java.nio.ByteBuffer;
public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Param ${type.from}Holder in;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = (${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
@@ -162,7 +169,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
// Re initialize the buffer everytime
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
out.scale = (int) scale.value;
out.precision = (int) precision.value;
@@ -182,18 +189,18 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
while(remainingScale > 0) {
- int power = (remainingScale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
+ int power = (remainingScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
int padding = 1;
if (power == 0) {
power = 9;
} else {
- padding = (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - power)));
+ padding = (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - power)));
}
- int mask = (int) org.apache.drill.common.util.DecimalUtility.getPowerOfTen(power);
+ int mask = (int) org.apache.drill.exec.util.DecimalUtility.getPowerOfTen(power);
- out.setInteger(index, (int) ((value % mask) * padding));
+ out.setInteger(index, (int) ((value % mask) * padding), out.start, out.buffer);
value = value/mask;
@@ -203,17 +210,17 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
}
while (value > 0) {
- out.setInteger(index, (int) (value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
- value = value/org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ out.setInteger(index, (int) (value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), out.start, out.buffer);
+ value = value/org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
index--;
}
// Round up or down the scale
if (in.scale != out.scale) {
- org.apache.drill.common.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
+ org.apache.drill.exec.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
}
// Set the sign
- out.setSign((in.value < 0));
+ out.setSign((in.value < 0), out.start, out.buffer);
}
}
@@ -226,6 +233,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -234,7 +243,9 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -255,7 +266,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
out.value = in.value;
// Truncate or pad additional zeroes if the output scale is different from input scale
- out.value = (${type.javatype}) (org.apache.drill.common.util.DecimalUtility.adjustScaleMultiply(out.value, (int) (out.scale - in.scale)));
+ out.value = (${type.javatype}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleMultiply(out.value, (int) (out.scale - in.scale)));
}
}
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
index 54232ea..acf2f1d 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
@@ -26,6 +26,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -33,9 +35,12 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -152,7 +157,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// Pad the number with zeroes if number of fractional digits is less than scale
if (fractionalDigits < scale.value) {
- out.value = (${type.javatype}) (org.apache.drill.common.util.DecimalUtility.adjustScaleMultiply(out.value, (int) (scale.value - fractionalDigits)));
+ out.value = (${type.javatype}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleMultiply(out.value, (int) (scale.value - fractionalDigits)));
}
// Negate the number if we saw a -ve sign
@@ -169,6 +174,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -176,9 +183,11 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -186,15 +195,14 @@ import java.nio.ByteBuffer;
public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = ${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize);
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
@@ -207,7 +215,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// Initialize the output buffer
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
int startIndex;
@@ -295,8 +303,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// Compute the number of slots needed in the ByteBuf to store the integer and fractional part
- int scaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(out.scale);
- int integerRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(integerDigits);
+ int scaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(out.scale);
+ int integerRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(integerDigits);
int ndigits = 0;
@@ -314,15 +322,15 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
next = (byte) Character.digit(next, radix);
- int value = (((int) org.apache.drill.common.util.DecimalUtility.getPowerOfTen(ndigits)) * next) + (out.getInteger(decimalBufferIndex));
- out.setInteger(decimalBufferIndex, value);
+ int value = (((int) org.apache.drill.exec.util.DecimalUtility.getPowerOfTen(ndigits)) * next) + (out.getInteger(decimalBufferIndex, out.start, out.buffer));
+ out.setInteger(decimalBufferIndex, value, out.start, out.buffer);
ndigits++;
/* We store the entire decimal as base 1 billion values, which has maximum of 9 digits (MAX_DIGITS)
* Once we have stored MAX_DIGITS in a given slot move to the next slot.
*/
- if (ndigits >= org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
+ if (ndigits >= org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
ndigits = 0;
decimalBufferIndex--;
}
@@ -336,7 +344,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
while (scaleIndex < scaleEndIndex) {
// check if we have scanned MAX_DIGITS and we need to move to the next index
- if (ndigits >= org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
+ if (ndigits >= org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
ndigits = 0;
decimalBufferIndex++;
}
@@ -352,8 +360,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
- int value = (out.getInteger(decimalBufferIndex) * radix) + next;
- out.setInteger(decimalBufferIndex, value);
+ int value = (out.getInteger(decimalBufferIndex, out.start, out.buffer) * radix) + next;
+ out.setInteger(decimalBufferIndex, value, out.start, out.buffer);
// added another digit to the current index
ndigits++;
@@ -374,29 +382,29 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
}
if (next > 4) {
// Need to round up
- out.setInteger(decimalBufferIndex, out.getInteger(decimalBufferIndex)+1);
+ out.setInteger(decimalBufferIndex, out.getInteger(decimalBufferIndex, out.start, out.buffer)+1, out.start, out.buffer);
}
}
// Pad zeroes in the fractional part so that number of digits = MAX_DIGITS
if (out.scale > 0) {
- int padding = (int) org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - ndigits));
- out.setInteger(decimalBufferIndex, out.getInteger(decimalBufferIndex) * padding);
+ int padding = (int) org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - ndigits));
+ out.setInteger(decimalBufferIndex, out.getInteger(decimalBufferIndex, out.start, out.buffer) * padding, out.start, out.buffer);
}
int carry = 0;
do {
// propogate the carry
- int tempValue = out.getInteger(decimalBufferIndex) + carry;
- if (tempValue >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- carry = tempValue / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
- tempValue = (tempValue % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
+ int tempValue = out.getInteger(decimalBufferIndex, out.start, out.buffer) + carry;
+ if (tempValue >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ carry = tempValue / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
+ tempValue = (tempValue % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
} else {
carry = 0;
}
- out.setInteger(decimalBufferIndex--, tempValue);
+ out.setInteger(decimalBufferIndex--, tempValue, out.start, out.buffer);
} while (carry > 0 && decimalBufferIndex >= 0);
}
- out.setSign(sign);
+ out.setSign(sign, out.start, out.buffer);
}
}
</#if> <#-- type.major -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
index 3a97b7b..05f5cb0 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-import org.apache.drill.exec.expr.annotations.Workspace;
<@pp.dropOutputFile />
@@ -36,12 +35,16 @@ import org.apache.drill.exec.expr.annotations.Workspace;
package org.apache.drill.exec.expr.fn.impl.gaggr;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillAggFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import io.netty.buffer.ByteBuf;
@@ -58,9 +61,11 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
@Param ${type.inputType}Holder in;
<#if aggrtype.funcName == "sum">
+ @Inject DrillBuf buffer;
@Workspace ObjectHolder value;
@Workspace IntHolder outputScale;
<#elseif type.outputType.endsWith("Sparse")>
+ @Inject DrillBuf buffer;
@Workspace ObjectHolder value;
<#else>
@Workspace ${type.runningType}Holder value;
@@ -76,21 +81,20 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value = new ObjectHolder();
${type.runningType}Holder tmp = new ${type.runningType}Holder();
value.obj = tmp;
- io.netty.buffer.ByteBuf buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[tmp.WIDTH]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ buffer = buffer.reallocIfNeeded(tmp.WIDTH);
tmp.buffer = buffer;
tmp.start = 0;
<#if aggrtype.funcName == "max">
for (int i = 0; i < tmp.nDecimalDigits; i++) {
- tmp.setInteger(i, 0xFFFFFFFF);
+ tmp.setInteger(i, 0xFFFFFFFF, tmp.start, tmp.buffer);
}
- tmp.setSign(true);
+ tmp.setSign(true, tmp.start, tmp.buffer);
<#elseif aggrtype.funcName == "min">
for (int i = 0; i < tmp.nDecimalDigits; i++) {
- tmp.setInteger(i, 0x7FFFFFFF);
+ tmp.setInteger(i, 0x7FFFFFFF, tmp.start, tmp.buffer);
}
// Set sign to be positive so initial value is maximum
- tmp.setSign(false);
+ tmp.setSign(false, tmp.start, tmp.buffer);
tmp.precision = ${type.runningType}Holder.maxPrecision;
</#if>
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
@@ -98,6 +102,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value.value = ${type.initValue};
</#if>
<#elseif aggrtype.funcName == "sum">
+ buffer = buffer.reallocIfNeeded(${type.outputType}Holder.WIDTH);
value = new ObjectHolder();
value.obj = java.math.BigDecimal.ZERO;
outputScale = new IntHolder();
@@ -120,13 +125,13 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#elseif aggrtype.funcName == "max">
<#if type.outputType.endsWith("Sparse")>
${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- int cmp = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(),
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(in.start, in.buffer),
in.scale, in.precision, tmp.buffer,
- tmp.start, tmp.getSign(), tmp.precision,
+ tmp.start, tmp.getSign(tmp.start, tmp.buffer), tmp.precision,
tmp.scale, in.WIDTH, in.nDecimalDigits, false);
if (cmp == 1) {
in.buffer.getBytes(in.start, tmp.buffer, 0, tmp.WIDTH);
- tmp.setSign(in.getSign());
+ tmp.setSign(in.getSign(in.start, in.buffer), tmp.start, tmp.buffer);
tmp.scale = in.scale;
tmp.precision = in.precision;
}
@@ -136,13 +141,13 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#elseif aggrtype.funcName == "min">
<#if type.outputType.endsWith("Sparse")>
${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- int cmp = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(),
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(in.start, in.buffer),
in.scale, in.precision, tmp.buffer,
- tmp.start, tmp.getSign(), tmp.precision,
+ tmp.start, tmp.getSign(tmp.start, tmp.buffer), tmp.precision,
tmp.scale, in.WIDTH, in.nDecimalDigits, false);
if (cmp == -1) {
in.buffer.getBytes(in.start, tmp.buffer, 0, tmp.WIDTH);
- tmp.setSign(in.getSign());
+ tmp.setSign(in.getSign(in.start, in.buffer), tmp.start, tmp.buffer);
tmp.scale = in.scale;
tmp.precision = in.precision;
}
@@ -151,9 +156,9 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
</#if>
<#elseif aggrtype.funcName == "sum">
<#if type.inputType.endsWith("Decimal9") || type.inputType.endsWith("Decimal18")>
- java.math.BigDecimal currentValue = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromPrimitiveTypes(in.value, in.scale, in.precision);
+ java.math.BigDecimal currentValue = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromPrimitiveTypes(in.value, in.scale, in.precision);
<#else>
- java.math.BigDecimal currentValue = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
+ java.math.BigDecimal currentValue = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
</#if>
value.obj = ((java.math.BigDecimal)(value.obj)).add(currentValue);
if (outputScale.value == Integer.MIN_VALUE) {
@@ -170,20 +175,18 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#if aggrtype.funcName == "count">
out.value = value.value;
<#elseif aggrtype.funcName == "sum">
- io.netty.buffer.ByteBuf buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[out.WIDTH]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
out.buffer = buffer;
out.start = 0;
out.scale = outputScale.value;
out.precision = 38;
value.obj = ((java.math.BigDecimal) (value.obj)).setScale(out.scale, java.math.BigDecimal.ROUND_HALF_UP);
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal((java.math.BigDecimal) value.obj, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal((java.math.BigDecimal) value.obj, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
<#else>
<#if type.outputType.endsWith("Sparse")>
${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
out.buffer = tmp.buffer;
out.start = tmp.start;
- out.setSign(tmp.getSign());
+ out.setSign(tmp.getSign(tmp.start, tmp.buffer), out.start, out.buffer);
out.scale = tmp.scale;
out.precision = tmp.precision;
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
@@ -204,18 +207,14 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value = new ObjectHolder();
${type.runningType}Holder tmp = new ${type.runningType}Holder();
value.obj = tmp;
- io.netty.buffer.ByteBuf buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[tmp.WIDTH]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
- tmp.buffer = buffer;
- tmp.start = 0;
for (int i = 0; i < tmp.nDecimalDigits; i++) {
- tmp.setInteger(i, 0xFFFFFFFF);
+ tmp.setInteger(i, 0xFFFFFFFF, tmp.start, tmp.buffer);
}
<#if aggrtype.funcName == "min">
// Set sign to be positive so initial value is maximum
- tmp.setSign(false);
+ tmp.setSign(false, tmp.start, tmp.buffer);
<#elseif aggrtype.funcName == "max">
- tmp.setSign(true);
+ tmp.setSign(true, tmp.start, tmp.buffer);
</#if>
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
value = new ${type.runningType}Holder();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions2.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions2.java
index b3f468b..7277e19 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions2.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions2.java
@@ -36,6 +36,8 @@ import org.apache.drill.exec.expr.annotations.Workspace;
package org.apache.drill.exec.expr.fn.impl.gaggr;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillAggFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
@@ -43,6 +45,8 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.*;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.record.RecordBatch;
import io.netty.buffer.ByteBuf;
@@ -55,12 +59,14 @@ public class Decimal${aggrtype.className}Functions {
public static class ${type.inputType}${aggrtype.className} implements DrillAggFunc{
@Param ${type.inputType}Holder in;
+ @Inject DrillBuf buffer;
@Workspace ObjectHolder value;
@Workspace ${type.countRunningType}Holder count;
@Workspace IntHolder outputScale;
@Output ${type.outputType}Holder out;
public void setup(RecordBatch b) {
+ buffer.reallocIfNeeded(${type.outputType}Holder.WIDTH);
value = new ObjectHolder();
value.obj = java.math.BigDecimal.ZERO;
count = new ${type.countRunningType}Holder();
@@ -80,9 +86,9 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
</#if>
count.value++;
<#if type.inputType.endsWith("Decimal9") || type.inputType.endsWith("Decimal18")>
- java.math.BigDecimal currentValue = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromPrimitiveTypes(in.value, in.scale, in.precision);
+ java.math.BigDecimal currentValue = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromPrimitiveTypes(in.value, in.scale, in.precision);
<#else>
- java.math.BigDecimal currentValue = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
+ java.math.BigDecimal currentValue = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
</#if>
value.obj = ((java.math.BigDecimal)(value.obj)).add(currentValue);
if (outputScale.value == Integer.MIN_VALUE) {
@@ -95,14 +101,12 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
@Override
public void output() {
- io.netty.buffer.ByteBuf buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[out.WIDTH]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
out.buffer = buffer;
out.start = 0;
out.scale = outputScale.value;
out.precision = 38;
java.math.BigDecimal average = ((java.math.BigDecimal)(value.obj)).divide(java.math.BigDecimal.valueOf(count.value, 0), out.scale, java.math.BigDecimal.ROUND_HALF_UP);
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(average, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(average, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
index 5278d9c..0c4af01 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
@@ -23,9 +23,9 @@ import org.apache.drill.exec.expr.annotations.Workspace;
<#macro compareBlock holderType left right absCompare output>
outside:{
- ${output} = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(left.buffer, left.start, left.getSign(),
+ ${output} = org.apache.drill.exec.util.DecimalUtility.compareSparseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer),
left.scale, left.precision, right.buffer,
- right.start, right.getSign(), right.precision,
+ right.start, right.getSign(right.start, right.buffer), right.precision,
right.scale, left.WIDTH, left.nDecimalDigits, ${absCompare});
}
@@ -33,19 +33,19 @@ import org.apache.drill.exec.expr.annotations.Workspace;
<#macro subtractBlock holderType in1 in2 result>
- int resultScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(result.scale);
+ int resultScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(result.scale);
int resultIndex = result.nDecimalDigits- 1;
- int leftScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(${in1}.scale);
- int leftIntRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(${in1}.precision - ${in1}.scale);
- int rightScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(${in2}.scale);
+ int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(${in1}.scale);
+ int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(${in1}.precision - ${in1}.scale);
+ int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(${in2}.scale);
int leftIndex = ${in1}.nDecimalDigits - 1;
int rightIndex = ${in2}.nDecimalDigits - 1;
/* If the left scale is bigger, simply copy over the digits into result */
while (leftScaleRoundedUp > rightScaleRoundedUp) {
- result.setInteger(resultIndex, ${in1}.getInteger(leftIndex));
+ result.setInteger(resultIndex, ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer), result.start, result.buffer);
leftIndex--;
resultIndex--;
leftScaleRoundedUp--;
@@ -55,14 +55,14 @@ import org.apache.drill.exec.expr.annotations.Workspace;
int carry = 0;
while(rightScaleRoundedUp > leftScaleRoundedUp) {
- int difference = 0 - ${in2}.getInteger(rightIndex) - carry;
+ int difference = 0 - ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer) - carry;
rightIndex--;
if (difference < 0) {
carry = 1;
- result.setInteger(resultIndex, (difference + org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ result.setInteger(resultIndex, (difference + org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
} else {
- result.setInteger(resultIndex, difference);
+ result.setInteger(resultIndex, difference, result.start, result.buffer);
carry = 0;
}
resultIndex--;
@@ -75,15 +75,15 @@ import org.apache.drill.exec.expr.annotations.Workspace;
*/
while (leftScaleRoundedUp > 0) {
- int difference = ${in1}.getInteger(leftIndex) - ${in2}.getInteger(rightIndex) - carry;
+ int difference = ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer) - ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer) - carry;
leftIndex--;
rightIndex--;
if (difference < 0) {
carry = 1;
- result.setInteger(resultIndex, (difference + org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ result.setInteger(resultIndex, (difference + org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
} else {
- result.setInteger(resultIndex, difference);
+ result.setInteger(resultIndex, difference, result.start, result.buffer);
carry = 0;
}
resultIndex--;
@@ -95,11 +95,11 @@ import org.apache.drill.exec.expr.annotations.Workspace;
*/
while(leftIntRoundedUp > 0) {
- int difference = ${in1}.getInteger(leftIndex);
+ int difference = ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer);
leftIndex--;
if (rightIndex >= 0) {
- difference -= ${in2}.getInteger(rightIndex);
+ difference -= ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer);
rightIndex--;
}
@@ -107,10 +107,10 @@ import org.apache.drill.exec.expr.annotations.Workspace;
if (difference < 0) {
carry = 1;
- result.setInteger(resultIndex, (difference + org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ result.setInteger(resultIndex, (difference + org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
} else {
carry = 0;
- result.setInteger(resultIndex, difference);
+ result.setInteger(resultIndex, difference, result.start, result.buffer);
}
resultIndex--;
leftIntRoundedUp--;
@@ -120,10 +120,10 @@ import org.apache.drill.exec.expr.annotations.Workspace;
<#macro addBlock holderType in1 in2 result>
- int resultScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(result.scale);
+ int resultScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(result.scale);
- int leftScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(${in1}.scale);
- int rightScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(${in2}.scale);
+ int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(${in1}.scale);
+ int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(${in2}.scale);
/* starting index for each decimal */
int leftIndex = ${in1}.nDecimalDigits - 1;
@@ -135,7 +135,7 @@ import org.apache.drill.exec.expr.annotations.Workspace;
*/
while (leftScaleRoundedUp > rightScaleRoundedUp) {
- result.setInteger(resultIndex, ${in1}.getInteger(leftIndex));
+ result.setInteger(resultIndex, ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer), result.start, result.buffer);
leftIndex--;
resultIndex--;
leftScaleRoundedUp--;
@@ -143,7 +143,7 @@ import org.apache.drill.exec.expr.annotations.Workspace;
}
while (rightScaleRoundedUp > leftScaleRoundedUp) {
- result.setInteger((resultIndex), ${in2}.getInteger(rightIndex));
+ result.setInteger((resultIndex), ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer), result.start, result.buffer);
rightIndex--;
resultIndex--;
rightScaleRoundedUp--;
@@ -155,15 +155,15 @@ import org.apache.drill.exec.expr.annotations.Workspace;
/* now the two scales are at the same level, we can add them */
while (resultScaleRoundedUp > 0) {
- sum += ${in1}.getInteger(leftIndex) + ${in2}.getInteger(rightIndex);
+ sum += ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer) + ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer);
leftIndex--;
rightIndex--;
- if (sum >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- result.setInteger(resultIndex, (sum - org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ if (sum >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ result.setInteger(resultIndex, (sum - org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
sum = 1;
} else {
- result.setInteger(resultIndex, sum);
+ result.setInteger(resultIndex, sum, result.start, result.buffer);
sum = 0;
}
resultIndex--;
@@ -173,49 +173,49 @@ import org.apache.drill.exec.expr.annotations.Workspace;
/* add the integer part */
while (leftIndex >= 0 && rightIndex >= 0) {
- sum += ${in1}.getInteger(leftIndex) + ${in2}.getInteger(rightIndex);
+ sum += ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer) + ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer);
leftIndex--;
rightIndex--;
- if (sum >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- result.setInteger(resultIndex, (sum - org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ if (sum >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ result.setInteger(resultIndex, (sum - org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
sum = 1;
} else {
- result.setInteger(resultIndex, sum);
+ result.setInteger(resultIndex, sum, result.start, result.buffer);
sum = 0;
}
resultIndex--;
}
while (resultIndex >= 0 && leftIndex >= 0) {
- sum += ${in1}.getInteger(leftIndex);
+ sum += ${in1}.getInteger(leftIndex, ${in1}.start, ${in1}.buffer);
leftIndex--;
- if (sum >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- result.setInteger(resultIndex, (sum - org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ if (sum >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ result.setInteger(resultIndex, (sum - org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
sum = 1;
} else {
- result.setInteger(resultIndex, sum);
+ result.setInteger(resultIndex, sum, result.start, result.buffer);
sum = 0;
}
}
while (resultIndex >= 0 && rightIndex >= 0) {
- sum += ${in2}.getInteger(rightIndex);
+ sum += ${in2}.getInteger(rightIndex, ${in2}.start, ${in2}.buffer);
rightIndex--;
- if (sum >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- result.setInteger(resultIndex, (sum - org.apache.drill.common.util.DecimalUtility.DIGITS_BASE));
+ if (sum >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ result.setInteger(resultIndex, (sum - org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE), result.start, result.buffer);
sum = 1;
} else {
- result.setInteger(resultIndex, sum);
+ result.setInteger(resultIndex, sum, result.start, result.buffer);
sum = 0;
}
}
/* store the last carry */
if (sum > 0)
- result.setInteger(resultIndex, sum);
+ result.setInteger(resultIndex, sum, result.start, result.buffer);
</#macro>
@@ -227,10 +227,10 @@ import org.apache.drill.exec.expr.annotations.Workspace;
int adjustment = 0;
if (left.scale < right.scale) {
- left.value = (${javaType}) (org.apache.drill.common.util.DecimalUtility.adjustScaleMultiply(left.value, (int) (right.scale - left.scale)));
+ left.value = (${javaType}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleMultiply(left.value, (int) (right.scale - left.scale)));
left.scale = right.scale;
} else if (right.scale < left.scale) {
- right.value = (${javaType}) (org.apache.drill.common.util.DecimalUtility.adjustScaleMultiply(right.value, (int) (left.scale - right.scale)));
+ right.value = (${javaType}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleMultiply(right.value, (int) (left.scale - right.scale)));
right.scale = left.scale;
}
</#macro>
@@ -244,6 +244,8 @@ import org.apache.drill.exec.expr.annotations.Workspace;
package org.apache.drill.exec.expr.fn.impl;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -252,7 +254,10 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -263,15 +268,14 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Workspace int outputScale;
@Workspace int outputPrecision;
@Output ${type.name}Holder result;
public void setup(RecordBatch incoming) {
- int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -287,13 +291,13 @@ public class ${type.name}Functions {
result.buffer = buffer;
result.start = 0;
- java.math.BigDecimal leftInput = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(left.buffer, left.start, left.nDecimalDigits, left.scale);
- java.math.BigDecimal rightInput = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(right.buffer, right.start, right.nDecimalDigits, right.scale);
+ java.math.BigDecimal leftInput = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(left.buffer, left.start, left.nDecimalDigits, left.scale);
+ java.math.BigDecimal rightInput = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(right.buffer, right.start, right.nDecimalDigits, right.scale);
java.math.BigDecimal addResult = leftInput.subtract(rightInput);
// set the scale
addResult.setScale(result.scale, java.math.BigDecimal.ROUND_HALF_UP);
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(addResult, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(addResult, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
}
}
@@ -304,13 +308,12 @@ public class ${type.name}Functions {
@Param ${type.name}Holder right;
@Workspace int outputScale;
@Workspace int outputPrecision;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.name}Holder result;
public void setup(RecordBatch incoming) {
- int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -326,13 +329,13 @@ public class ${type.name}Functions {
result.buffer = buffer;
result.start = 0;
- java.math.BigDecimal leftInput = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(left.buffer, left.start, left.nDecimalDigits, left.scale);
- java.math.BigDecimal rightInput = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(right.buffer, right.start, right.nDecimalDigits, right.scale);
+ java.math.BigDecimal leftInput = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(left.buffer, left.start, left.nDecimalDigits, left.scale);
+ java.math.BigDecimal rightInput = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(right.buffer, right.start, right.nDecimalDigits, right.scale);
java.math.BigDecimal addResult = leftInput.add(rightInput);
// set the scale
addResult.setScale(result.scale, java.math.BigDecimal.ROUND_HALF_UP);
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(addResult, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(addResult, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
}
}
@@ -341,16 +344,15 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Workspace int[] tempResult;
@Workspace int outputScale;
@Workspace int outputPrecision;
@Output ${type.name}Holder result;
public void setup(RecordBatch incoming) {
- int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
tempResult = new int[${type.storage} * ${type.storage}];
outputPrecision = Integer.MIN_VALUE;
}
@@ -375,10 +377,10 @@ public class ${type.name}Functions {
// Remove the leading zeroes from the integer part of the input
int leftIndex = 0;
- int leftStopIndex = left.nDecimalDigits - org.apache.drill.common.util.DecimalUtility.roundUp(left.scale);
+ int leftStopIndex = left.nDecimalDigits - org.apache.drill.exec.util.DecimalUtility.roundUp(left.scale);
while (leftIndex < leftStopIndex) {
- if (left.getInteger(leftIndex) > 0)
+ if (left.getInteger(leftIndex, left.start, left.buffer) > 0)
break;
leftIndex++;
}
@@ -387,10 +389,10 @@ public class ${type.name}Functions {
/* Remove the leaing zeroes from the integer part of the input */
int rightIndex = 0;
- int rightStopIndex = right.nDecimalDigits - org.apache.drill.common.util.DecimalUtility.roundUp(right.scale);
+ int rightStopIndex = right.nDecimalDigits - org.apache.drill.exec.util.DecimalUtility.roundUp(right.scale);
while(rightIndex < rightStopIndex) {
- if (right.getInteger(rightIndex) > 0)
+ if (right.getInteger(rightIndex, right.start, right.buffer) > 0)
break;
rightIndex++;
}
@@ -398,7 +400,7 @@ public class ${type.name}Functions {
int rightIntegerSize = rightStopIndex - rightIndex;
int resultIntegerSize = leftIntegerSize + rightIntegerSize;
- int resultScaleSize = org.apache.drill.common.util.DecimalUtility.roundUp(left.scale + right.scale);
+ int resultScaleSize = org.apache.drill.exec.util.DecimalUtility.roundUp(left.scale + right.scale);
int leftSize = left.nDecimalDigits - 1;
int rightSize = right.nDecimalDigits - 1;
@@ -413,13 +415,13 @@ public class ${type.name}Functions {
for (int j = rightSize; j >= rightIndex; j--) {
- long mulResult = (long) right.getInteger(j) * (long) left.getInteger(i);
+ long mulResult = (long) right.getInteger(j, right.start, right.buffer) * (long) left.getInteger(i, left.start, left.buffer);
long tempSum = tempResult[currentIndex] + mulResult + carry;
- if (tempSum >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- tempResult[currentIndex] = (int) (tempSum % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- carry = (int) (tempSum / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
+ if (tempSum >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ tempResult[currentIndex] = (int) (tempSum % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
+ carry = (int) (tempSum / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
} else {
tempResult[currentIndex] = (int) tempSum;
carry = 0;
@@ -437,7 +439,7 @@ public class ${type.name}Functions {
/* We have computed the result of the multiplication; check if we need to
* round a portion of the fractional part
*/
- resultScaleSize = org.apache.drill.common.util.DecimalUtility.roundUp(result.scale);
+ resultScaleSize = org.apache.drill.exec.util.DecimalUtility.roundUp(result.scale);
if (result.scale < (left.scale + right.scale)) {
/* The scale of the output data type is lesser than the scale
@@ -447,11 +449,11 @@ public class ${type.name}Functions {
int lastScaleIndex = currentIndex + resultIntegerSize + resultScaleSize - 1;
// compute the power of 10 necessary to find if we need to round up
- int roundFactor = (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen(
- org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - ((result.scale + 1) % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS)));
+ int roundFactor = (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen(
+ org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - ((result.scale + 1) % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS)));
// index of rounding digit
- int roundIndex = currentIndex + resultIntegerSize + org.apache.drill.common.util.DecimalUtility.roundUp(result.scale + 1) - 1;
+ int roundIndex = currentIndex + resultIntegerSize + org.apache.drill.exec.util.DecimalUtility.roundUp(result.scale + 1) - 1;
// Check the first chopped digit to see if we need to round up
int carry = ((tempResult[roundIndex] / roundFactor) % 10) > 4 ? 1 : 0;
@@ -459,8 +461,8 @@ public class ${type.name}Functions {
if (result.scale > 0) {
// Compute the power of 10 necessary to chop of the fractional part
- int scaleFactor = (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen(
- org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (result.scale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS)));
+ int scaleFactor = (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen(
+ org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (result.scale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS)));
// Chop the unwanted fractional part
tempResult[lastScaleIndex] /= scaleFactor;
tempResult[lastScaleIndex] *= scaleFactor;
@@ -472,9 +474,9 @@ public class ${type.name}Functions {
// propogate the carry
while (carry > 0 && lastScaleIndex >= 0) {
int tempSum = tempResult[lastScaleIndex] + carry;
- if (tempSum >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- tempResult[lastScaleIndex] = (tempSum % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- carry = (int) (tempSum / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
+ if (tempSum >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ tempResult[lastScaleIndex] = (tempSum % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
+ carry = (int) (tempSum / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
} else {
tempResult[lastScaleIndex] = tempSum;
carry = 0;
@@ -496,14 +498,14 @@ public class ${type.name}Functions {
int outputIndex = result.nDecimalDigits - 1;
for (int i = (currentIndex + resultIntegerSize + resultScaleSize - 1); i >= currentIndex; i--) {
- result.setInteger(outputIndex--, tempResult[i]);
+ result.setInteger(outputIndex--, tempResult[i], result.start, result.buffer);
}
// Set the remaining digits to be zero
while(outputIndex >= 0) {
- result.setInteger(outputIndex--, 0);
+ result.setInteger(outputIndex--, 0, result.start, result.buffer);
}
- result.setSign(left.getSign() != right.getSign());
+ result.setSign(left.getSign(left.start, left.buffer) != right.getSign(right.start, right.buffer), result.start, result.buffer);
}
}
@@ -513,14 +515,13 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
@Output ${type.name}Holder result;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Workspace int outputScale;
@Workspace int outputPrecision;
public void setup(RecordBatch incoming) {
- int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -536,12 +537,12 @@ public class ${type.name}Functions {
result.buffer = buffer;
result.start = 0;
- java.math.BigDecimal numerator = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(left.buffer, left.start, left.nDecimalDigits, left.scale, true);
- java.math.BigDecimal denominator = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(right.buffer, right.start, right.nDecimalDigits, right.scale, true);
+ java.math.BigDecimal numerator = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(left.buffer, left.start, left.nDecimalDigits, left.scale, true);
+ java.math.BigDecimal denominator = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(right.buffer, right.start, right.nDecimalDigits, right.scale, true);
java.math.BigDecimal output = numerator.divide(denominator, (int) result.scale, java.math.BigDecimal.ROUND_HALF_UP);
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(output, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(output, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
}
}
@@ -551,14 +552,13 @@ public class ${type.name}Functions {
@Param ${type.name}Holder left;
@Param ${type.name}Holder right;
@Output ${type.name}Holder result;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Workspace int outputScale;
@Workspace int outputPrecision;
public void setup(RecordBatch incoming) {
- int size = (${type.storage} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -574,13 +574,13 @@ public class ${type.name}Functions {
result.buffer = buffer;
result.start = 0;
- java.math.BigDecimal numerator = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(left.buffer, left.start, left.nDecimalDigits, left.scale, true);
- java.math.BigDecimal denominator = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(right.buffer, right.start, right.nDecimalDigits, right.scale, true);
+ java.math.BigDecimal numerator = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(left.buffer, left.start, left.nDecimalDigits, left.scale, true);
+ java.math.BigDecimal denominator = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(right.buffer, right.start, right.nDecimalDigits, right.scale, true);
java.math.BigDecimal output = numerator.remainder(denominator);
output.setScale(result.scale, java.math.BigDecimal.ROUND_HALF_UP);
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(output, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(output, result.buffer, result.start, result.scale, result.precision, result.nDecimalDigits);
}
}
@@ -616,11 +616,11 @@ public class ${type.name}Functions {
boolean zeroValue = true;
- if (in.getSign() == true) {
+ if (in.getSign(in.start, in.buffer) == true) {
out.value = -1;
} else {
for (int i = 0; i < ${type.storage}; i++) {
- if (in.getInteger(i) != 0) {
+ if (in.getInteger(i, in.start, in.buffer) != 0) {
zeroValue = false;
break;
}
@@ -644,18 +644,18 @@ public class ${type.name}Functions {
out.precision = in.precision;
out.buffer = in.buffer;
out.start = in.start;
- boolean sign = in.getSign();
+ boolean sign = in.getSign(in.start, in.buffer);
// Indicates whether we need to add 1 to the integer part, while performing ceil
int carry = 0;
- int scaleStartIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale);
+ int scaleStartIndex = ${type.storage} - org.apache.drill.exec.util.DecimalUtility.roundUp(in.scale);
int srcIntIndex = scaleStartIndex - 1;
if (sign == false) {
// For negative values ceil we don't need to increment the integer part
while (scaleStartIndex < ${type.storage}) {
- if (out.getInteger(scaleStartIndex) != 0) {
+ if (out.getInteger(scaleStartIndex, out.start, out.buffer) != 0) {
carry = 1;
break;
}
@@ -666,12 +666,12 @@ public class ${type.name}Functions {
// Truncate the fractional part, move the integer part
int destIndex = ${type.storage} - 1;
while (srcIntIndex >= 0) {
- out.setInteger(destIndex--, out.getInteger(srcIntIndex--));
+ out.setInteger(destIndex--, out.getInteger(srcIntIndex--, out.start, out.buffer), out.start, out.buffer);
}
// Set the remaining portion of the decimal to be zeroes
while (destIndex >= 0) {
- out.setInteger(destIndex--, 0);
+ out.setInteger(destIndex--, 0, out.start, out.buffer);
}
// Add the carry
@@ -679,20 +679,20 @@ public class ${type.name}Functions {
destIndex = ${type.storage} - 1;
while (destIndex >= 0) {
- int intValue = out.getInteger(destIndex);
+ int intValue = out.getInteger(destIndex, out.start, out.buffer);
intValue += carry;
- if (intValue >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- out.setInteger(destIndex--, intValue % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- carry = intValue / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ if (intValue >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ out.setInteger(destIndex--, intValue % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE, out.start, out.buffer);
+ carry = intValue / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
} else {
- out.setInteger(destIndex--, intValue);
+ out.setInteger(destIndex--, intValue, out.start, out.buffer);
break;
}
}
}
// set the sign
- out.setSign(sign);
+ out.setSign(sign, out.start, out.buffer);
}
}
@@ -710,18 +710,18 @@ public class ${type.name}Functions {
out.precision = in.precision;
out.buffer = in.buffer;
out.start = in.start;
- boolean sign = in.getSign();
+ boolean sign = in.getSign(in.start, in.buffer);
// Indicates whether we need to decrement 1 from the integer part, while performing floor, done for -ve values
int carry = 0;
- int scaleStartIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale);
+ int scaleStartIndex = ${type.storage} - org.apache.drill.exec.util.DecimalUtility.roundUp(in.scale);
int srcIntIndex = scaleStartIndex - 1;
if (sign == true) {
// For negative values ceil we don't need to increment the integer part
while (scaleStartIndex < ${type.storage}) {
- if (out.getInteger(scaleStartIndex) != 0) {
+ if (out.getInteger(scaleStartIndex, out.start, out.buffer) != 0) {
carry = 1;
break;
}
@@ -732,32 +732,32 @@ public class ${type.name}Functions {
// Truncate the fractional part, move the integer part
int destIndex = ${type.storage} - 1;
while (srcIntIndex >= 0) {
- out.setInteger(destIndex--, out.getInteger(srcIntIndex--));
+ out.setInteger(destIndex--, out.getInteger(srcIntIndex--, out.start, out.buffer), out.start, out.buffer);
}
// Set the remaining portion of the decimal to be zeroes
while (destIndex >= 0) {
- out.setInteger(destIndex--, 0);
+ out.setInteger(destIndex--, 0, out.start, out.buffer);
}
// Add the carry
if (carry != 0) {
destIndex = ${type.storage} - 1;
while (destIndex >= 0) {
- int intValue = out.getInteger(destIndex);
+ int intValue = out.getInteger(destIndex, out.start, out.buffer);
intValue += carry;
- if (intValue >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- out.setInteger(destIndex--, intValue % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- carry = intValue / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ if (intValue >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ out.setInteger(destIndex--, intValue % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE, out.start, out.buffer);
+ carry = intValue / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
} else {
- out.setInteger(destIndex--, intValue);
+ out.setInteger(destIndex--, intValue, out.start, out.buffer);
break;
}
}
}
// set the sign
- out.setSign(sign);
+ out.setSign(sign, out.start, out.buffer);
}
}
@@ -775,23 +775,23 @@ public class ${type.name}Functions {
out.precision = in.precision;
out.buffer = in.buffer;
out.start = in.start;
- boolean sign = in.getSign();
+ boolean sign = in.getSign(in.start, in.buffer);
// Integer part's src index
- int srcIntIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale) - 1;
+ int srcIntIndex = ${type.storage} - org.apache.drill.exec.util.DecimalUtility.roundUp(in.scale) - 1;
// Truncate the fractional part, move the integer part
int destIndex = ${type.storage} - 1;
while (srcIntIndex >= 0) {
- out.setInteger(destIndex--, out.getInteger(srcIntIndex--));
+ out.setInteger(destIndex--, out.getInteger(srcIntIndex--, out.start, out.buffer), out.start, out.buffer);
}
// Set the remaining portion of the decimal to be zeroes
while (destIndex >= 0) {
- out.setInteger(destIndex--, 0);
+ out.setInteger(destIndex--, 0, out.start, out.buffer);
}
// set the sign
- out.setSign(sign);
+ out.setSign(sign, out.start, out.buffer);
}
}
@@ -810,10 +810,10 @@ public class ${type.name}Functions {
result.precision = left.precision;
result.buffer = left.buffer;
result.start = left.start;
- boolean sign = left.getSign();
+ boolean sign = left.getSign(left.start, left.buffer);
- int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(right.value);
- int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(left.scale);
+ int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(right.value);
+ int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(left.scale);
if (right.value < left.scale) {
// Get the source index beyond which we will truncate
@@ -824,22 +824,22 @@ public class ${type.name}Functions {
int destIndex = ${type.storage} - 1;
if (srcIndex != destIndex) {
while (srcIndex >= 0) {
- result.setInteger(destIndex--, result.getInteger(srcIndex--));
+ result.setInteger(destIndex--, result.getInteger(srcIndex--, result.start, result.buffer), result.start, result.buffer);
}
// Set the remaining portion of the decimal to be zeroes
while (destIndex >= 0) {
- result.setInteger(destIndex--, 0);
+ result.setInteger(destIndex--, 0, result.start, result.buffer);
}
}
// We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
- int truncateFactor = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (right.value % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS);
- if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) {
- truncateFactor = (int) org.apache.drill.common.util.DecimalUtility.getPowerOfTen(truncateFactor);
- int fractionalDigits = result.getInteger(${type.storage} - 1);
+ int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (right.value % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ truncateFactor = (int) org.apache.drill.exec.util.DecimalUtility.getPowerOfTen(truncateFactor);
+ int fractionalDigits = result.getInteger(${type.storage} - 1, result.start, result.buffer);
fractionalDigits /= truncateFactor;
- result.setInteger(${type.storage} - 1, fractionalDigits * truncateFactor);
+ result.setInteger(${type.storage} - 1, fractionalDigits * truncateFactor, result.start, result.buffer);
}
} else if (right.value > left.scale) {
// Add fractional digits to the decimal
@@ -851,24 +851,24 @@ public class ${type.name}Functions {
// Check while extending scale, we are not overwriting integer part
while (srcIndex < destIndex) {
- if (result.getInteger(srcIndex++) != 0) {
+ if (result.getInteger(srcIndex++, result.start, result.buffer) != 0) {
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
}
}
srcIndex = 0;
while (destIndex < ${type.storage}) {
- result.setInteger(srcIndex++, result.getInteger(destIndex++));
+ result.setInteger(srcIndex++, result.getInteger(destIndex++, result.start, result.buffer), result.start, result.buffer);
}
// Clear the remaining part
while (srcIndex < ${type.storage}) {
- result.setInteger(srcIndex++, 0);
+ result.setInteger(srcIndex++, 0, result.start, result.buffer);
}
}
}
// set the sign
- result.setSign(sign);
+ result.setSign(sign, result.start, result.buffer);
}
}
@@ -886,15 +886,15 @@ public class ${type.name}Functions {
out.precision = in.precision;
out.buffer = in.buffer;
out.start = in.start;
- boolean sign = in.getSign();
+ boolean sign = in.getSign(in.start, in.buffer);
boolean roundUp = false;
// Get the first fractional digit to see if want to round up or not
- int scaleIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale);
+ int scaleIndex = ${type.storage} - org.apache.drill.exec.util.DecimalUtility.roundUp(in.scale);
if (scaleIndex < ${type.storage}) {
- int fractionalPart = out.getInteger(scaleIndex);
- int digit = fractionalPart / (org.apache.drill.common.util.DecimalUtility.DIGITS_BASE / 10);
+ int fractionalPart = out.getInteger(scaleIndex, out.start, out.buffer);
+ int digit = fractionalPart / (org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE / 10);
if (digit > 4) {
roundUp = true;
@@ -907,30 +907,30 @@ public class ${type.name}Functions {
// Truncate the fractional part, move the integer part
int destIndex = ${type.storage} - 1;
while (srcIntIndex >= 0) {
- out.setInteger(destIndex--, out.getInteger(srcIntIndex--));
+ out.setInteger(destIndex--, out.getInteger(srcIntIndex--, out.start, out.buffer), out.start, out.buffer);
}
// Set the remaining portion of the decimal to be zeroes
while (destIndex >= 0) {
- out.setInteger(destIndex--, 0);
+ out.setInteger(destIndex--, 0, out.start, out.buffer);
}
// Perform the roundup
srcIntIndex = ${type.storage} - 1;
if (roundUp == true) {
while (srcIntIndex >= 0) {
- int value = out.getInteger(srcIntIndex) + 1;
- if (value >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) {
- out.setInteger(srcIntIndex--, value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE);
- value = value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ int value = out.getInteger(srcIntIndex, out.start, out.buffer) + 1;
+ if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ out.setInteger(srcIntIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE, out.start, out.buffer);
+ value = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
} else {
- out.setInteger(srcIntIndex--, value);
+ out.setInteger(srcIntIndex--, value, out.start, out.buffer);
break;
}
}
}
// set the sign
- out.setSign(sign);
+ out.setSign(sign, out.start, out.buffer);
}
}
@@ -949,11 +949,11 @@ public class ${type.name}Functions {
result.precision = left.precision;
result.buffer = left.buffer;
result.start = left.start;
- boolean sign = left.getSign();
+ boolean sign = left.getSign(left.start, left.buffer);
- org.apache.drill.common.util.DecimalUtility.roundDecimal(result.buffer, result.start, result.nDecimalDigits, result.scale, left.scale);
+ org.apache.drill.exec.util.DecimalUtility.roundDecimal(result.buffer, result.start, result.nDecimalDigits, result.scale, left.scale);
// set the sign
- result.setSign(sign);
+ result.setSign(sign, result.start, result.buffer);
}
}
@@ -1069,6 +1069,8 @@ public class ${type.name}Functions {
package org.apache.drill.exec.expr.fn.impl;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -1076,7 +1078,9 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -1092,7 +1096,7 @@ public class ${type.name}Functions {
public void setup(RecordBatch incoming) {}
public void eval() {
- out.value = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ out.value = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
}
}
@@ -1105,7 +1109,7 @@ public class ${type.name}Functions {
public void setup(RecordBatch incoming) {}
public void eval() {
- int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
out.value = cmp == -1 ? 1 : 0;
}
}
@@ -1119,7 +1123,7 @@ public class ${type.name}Functions {
public void setup(RecordBatch incoming) {}
public void eval() {
- int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
out.value = cmp < 1 ? 1 : 0;
}
}
@@ -1133,7 +1137,7 @@ public class ${type.name}Functions {
public void setup(RecordBatch incoming) {}
public void eval() {
- int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
out.value = cmp == 1 ? 1 : 0;
}
}
@@ -1147,7 +1151,7 @@ public class ${type.name}Functions {
public void setup(RecordBatch incoming) {}
public void eval() {
- int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
out.value = cmp > -1 ? 1 : 0;
}
}
@@ -1161,7 +1165,7 @@ public class ${type.name}Functions {
public void setup(RecordBatch incoming) {}
public void eval() {
- int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
out.value = cmp == 0 ? 1 : 0;
}
}
@@ -1177,7 +1181,7 @@ public class ${type.name}Functions {
public void eval() {
- int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH);
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(left.start, left.buffer), right.buffer, right.start, right.getSign(right.start, right.buffer), left.WIDTH);
out.value = cmp != 0 ? 1 : 0;
}
}
@@ -1190,6 +1194,8 @@ public class ${type.name}Functions {
package org.apache.drill.exec.expr.fn.impl;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -1198,7 +1204,9 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -1395,7 +1403,7 @@ public class ${type.name}Functions {
public void eval() {
- out.value =(${type.storage}) (org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(in.value, (int) in.scale));
+ out.value =(${type.storage}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(in.value, (int) in.scale));
out.precision = out.maxPrecision;
out.scale = 0;
}
@@ -1412,7 +1420,7 @@ public class ${type.name}Functions {
public void eval() {
- out.value = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(left.value, (int) (left.scale - right.value)));
+ out.value = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(left.value, (int) (left.scale - right.value)));
out.precision = out.maxPrecision;
out.scale = right.value;
}
@@ -1429,7 +1437,7 @@ public class ${type.name}Functions {
}
public void eval() {
- ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale));
+ ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) in.scale));
// Get the integer part
${type.storage} integerPart = in.value / scaleFactor;
@@ -1456,7 +1464,7 @@ public class ${type.name}Functions {
public void eval() {
- ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale));
+ ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) in.scale));
out.scale = 0;
out.value = (in.value / scaleFactor);
@@ -1481,7 +1489,7 @@ public class ${type.name}Functions {
public void eval() {
- ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale));
+ ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) in.scale));
${type.storage} extractDigit = scaleFactor / 10;
out.scale = 0;
@@ -1516,14 +1524,14 @@ public class ${type.name}Functions {
public void eval() {
- ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) left.scale));
- ${type.storage} newScaleFactor = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) right.value));
- ${type.storage} truncScaleFactor = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen( Math.abs(left.scale - right.value)));
+ ${type.storage} scaleFactor = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) left.scale));
+ ${type.storage} newScaleFactor = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) right.value));
+ ${type.storage} truncScaleFactor = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen( Math.abs(left.scale - right.value)));
int truncFactor = (int) (left.scale - right.value);
// If rounding scale is >= current scale
if (right.value >= left.scale) {
- out.value = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.adjustScaleMultiply(left.value, (int) (right.value - left.scale)));
+ out.value = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleMultiply(left.value, (int) (right.value - left.scale)));
}
else {
out.scale = right.value;
@@ -1534,12 +1542,12 @@ public class ${type.name}Functions {
${type.storage} fractionalPart = left.value % scaleFactor;
// From the entire fractional part extract the digits upto which rounding is needed
- ${type.storage} newFractionalPart = (${type.storage}) (org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(fractionalPart, truncFactor));
+ ${type.storage} newFractionalPart = (${type.storage}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(fractionalPart, truncFactor));
${type.storage} truncatedFraction = fractionalPart % truncScaleFactor;
// Get the truncated fractional part and extract the first digit to see if we need to add 1
- int digit = Math.abs((int) org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(truncatedFraction, truncFactor - 1));
+ int digit = Math.abs((int) org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(truncatedFraction, truncFactor - 1));
if (digit > 4) {
if (left.value > 0) {
[31/47] git commit: Support for @inject in aggregation functions
Posted by ja...@apache.org.
Support for @inject in aggregation functions
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a99aecad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a99aecad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a99aecad
Branch: refs/heads/master
Commit: a99aecad75e1e106adf510274d2bbc642108b167
Parents: 7fc8d6d
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Aug 26 02:53:46 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 09:08:42 2014 -0700
----------------------------------------------------------------------
.../Decimal/DecimalAggrTypeFunctions1.java | 93 +++++++++++---------
.../templates/VarCharAggrFunctions1.java | 34 +++----
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 40 ++++++---
.../exec/expr/fn/impl/ByteFunctionHelpers.java | 67 +++++++++++++-
.../drill/exec/expr/fn/impl/DrillByteArray.java | 60 +++++++++++++
.../expr/fn/impl/StringFunctionHelpers.java | 2 +-
.../apache/drill/exec/util/DecimalUtility.java | 31 +++++++
.../drill/jdbc/test/TestFunctionsQuery.java | 1 +
8 files changed, 258 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
index 05f5cb0..54cec10 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
@@ -66,6 +66,8 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
@Workspace IntHolder outputScale;
<#elseif type.outputType.endsWith("Sparse")>
@Inject DrillBuf buffer;
+ @Workspace IntHolder scale;
+ @Workspace IntHolder precision;
@Workspace ObjectHolder value;
<#else>
@Workspace ${type.runningType}Holder value;
@@ -78,24 +80,24 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value.value = 0;
<#elseif aggrtype.funcName == "max" || aggrtype.funcName == "min">
<#if type.outputType.endsWith("Sparse")>
+ scale.value = 0;
+ precision.value = 0;
value = new ObjectHolder();
- ${type.runningType}Holder tmp = new ${type.runningType}Holder();
+ //${type.runningType}Holder tmp = new ${type.runningType}Holder();
+ byte[] byteArray = new byte[${type.runningType}Holder.WIDTH];
+ org.apache.hadoop.io.Text tmp = new org.apache.hadoop.io.Text(byteArray);
value.obj = tmp;
- buffer = buffer.reallocIfNeeded(tmp.WIDTH);
- tmp.buffer = buffer;
- tmp.start = 0;
<#if aggrtype.funcName == "max">
- for (int i = 0; i < tmp.nDecimalDigits; i++) {
- tmp.setInteger(i, 0xFFFFFFFF, tmp.start, tmp.buffer);
+ for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setInteger(tmp.getBytes(), i, 0xFFFFFFFF);
}
- tmp.setSign(true, tmp.start, tmp.buffer);
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setSign(tmp.getBytes(), true);
<#elseif aggrtype.funcName == "min">
- for (int i = 0; i < tmp.nDecimalDigits; i++) {
- tmp.setInteger(i, 0x7FFFFFFF, tmp.start, tmp.buffer);
+ for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setInteger(tmp.getBytes(), i, 0x7FFFFFFF);
}
// Set sign to be positive so initial value is maximum
- tmp.setSign(false, tmp.start, tmp.buffer);
- tmp.precision = ${type.runningType}Holder.maxPrecision;
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setSign(tmp.getBytes(), false);
</#if>
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
value = new ${type.runningType}Holder();
@@ -124,32 +126,34 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value.value++;
<#elseif aggrtype.funcName == "max">
<#if type.outputType.endsWith("Sparse")>
- ${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(in.start, in.buffer),
- in.scale, in.precision, tmp.buffer,
- tmp.start, tmp.getSign(tmp.start, tmp.buffer), tmp.precision,
- tmp.scale, in.WIDTH, in.nDecimalDigits, false);
+ //${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
+ org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseSamePrecScale(in.buffer, in.start, tmp.getBytes(), tmp.getLength());
if (cmp == 1) {
- in.buffer.getBytes(in.start, tmp.buffer, 0, tmp.WIDTH);
- tmp.setSign(in.getSign(in.start, in.buffer), tmp.start, tmp.buffer);
- tmp.scale = in.scale;
- tmp.precision = in.precision;
+ //in.buffer.getBytes(in.start, tmp.getBytes(), 0, ${type.runningType}Holder.WIDTH);
+ for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setInteger(tmp.getBytes(), i, ${type.runningType}Holder.getInteger(i, in.start, in.buffer));
+ }
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setSign(tmp.getBytes(), in.getSign(in.start, in.buffer));
+ scale.value = in.scale;
+ precision.value = in.precision;
}
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
value.value = Math.max(value.value, in.value);
</#if>
<#elseif aggrtype.funcName == "min">
<#if type.outputType.endsWith("Sparse")>
- ${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(in.start, in.buffer),
- in.scale, in.precision, tmp.buffer,
- tmp.start, tmp.getSign(tmp.start, tmp.buffer), tmp.precision,
- tmp.scale, in.WIDTH, in.nDecimalDigits, false);
+ //${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
+ org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseSamePrecScale(in.buffer, in.start, tmp.getBytes(), tmp.getLength());
if (cmp == -1) {
- in.buffer.getBytes(in.start, tmp.buffer, 0, tmp.WIDTH);
- tmp.setSign(in.getSign(in.start, in.buffer), tmp.start, tmp.buffer);
- tmp.scale = in.scale;
- tmp.precision = in.precision;
+ //in.buffer.getBytes(in.start, tmp.getBytes(), 0, ${type.runningType}Holder.WIDTH);
+ for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setInteger(tmp.getBytes(), i, ${type.runningType}Holder.getInteger(i, in.start, in.buffer));
+ }
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setSign(tmp.getBytes(), in.getSign(in.start, in.buffer));
+ scale.value = in.scale;
+ precision.value = in.precision;
}
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
value.value = Math.min(value.value, in.value);
@@ -183,12 +187,17 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal((java.math.BigDecimal) value.obj, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
<#else>
<#if type.outputType.endsWith("Sparse")>
- ${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- out.buffer = tmp.buffer;
- out.start = tmp.start;
- out.setSign(tmp.getSign(tmp.start, tmp.buffer), out.start, out.buffer);
- out.scale = tmp.scale;
- out.precision = tmp.precision;
+ org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ buffer = buffer.reallocIfNeeded(tmp.getLength());
+ //buffer.setBytes(0, tmp.getBytes(), 0, tmp.getLength());
+ for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
+ ${type.runningType}Holder.setInteger(i, org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.getInteger(tmp.getBytes(), i), 0, buffer);
+ }
+ out.buffer = buffer;
+ out.start = 0;
+ out.setSign(org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.getSign(tmp.getBytes()), out.start, out.buffer);
+ out.scale = scale.value;
+ out.precision = precision.value;
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
out.value = value.value;
out.scale = value.scale;
@@ -204,17 +213,17 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value.value = 0;
<#elseif aggrtype.funcName == "max" || aggrtype.funcName == "min">
<#if type.outputType.endsWith("Sparse")>
- value = new ObjectHolder();
- ${type.runningType}Holder tmp = new ${type.runningType}Holder();
- value.obj = tmp;
- for (int i = 0; i < tmp.nDecimalDigits; i++) {
- tmp.setInteger(i, 0xFFFFFFFF, tmp.start, tmp.buffer);
+ org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setInteger(tmp.getBytes(), i, 0xFFFFFFFF);
}
+ scale.value = 0;
+ precision.value = 0;
<#if aggrtype.funcName == "min">
// Set sign to be positive so initial value is maximum
- tmp.setSign(false, tmp.start, tmp.buffer);
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setSign(tmp.getBytes(), false);
<#elseif aggrtype.funcName == "max">
- tmp.setSign(true, tmp.start, tmp.buffer);
+ org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setSign(tmp.getBytes(), true);
</#if>
<#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18">
value = new ${type.runningType}Holder();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java b/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
index 4b9c43b..ea56624 100644
--- a/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
@@ -74,10 +74,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
init = new UInt1Holder();
init.value = 0;
value = new ObjectHolder();
- ${type.runningType}Holder tmp = new ${type.runningType}Holder();
- tmp.start = 0;
- tmp.end = 0;
- tmp.buffer = null;
+ org.apache.drill.exec.expr.fn.impl.DrillByteArray tmp = new org.apache.drill.exec.expr.fn.impl.DrillByteArray();
value.obj = tmp;
<#else>
@@ -96,7 +93,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
}
</#if>
<#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
- ${type.runningType}Holder tmp = (${type.runningType}Holder)value.obj;
+ org.apache.drill.exec.expr.fn.impl.DrillByteArray tmp = (org.apache.drill.exec.expr.fn.impl.DrillByteArray) value.obj;
int cmp = 0;
boolean swap = false;
@@ -106,7 +103,8 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
swap = true;
} else {
// Compare the bytes
- int compare = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.compare(in.buffer.memoryAddress(), in.start, in.end, tmp.buffer.memoryAddress(), tmp.start, tmp.end);
+ cmp = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.compare(in.buffer.memoryAddress(), in.start, in.end, tmp.getBytes(), 0, tmp.getLength());
+
<#if aggrtype.className == "Min">
swap = (cmp == -1);
@@ -115,11 +113,15 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
</#if>
}
if (swap) {
- int length = in.end - in.start;
- this.buf = tmp.buffer = buf.reallocIfNeeded(in.end - in.start);
- in.buffer.getBytes(in.start, tmp.buffer, 0, length);
- tmp.end = length;
-
+ int inputLength = in.end - in.start;
+ if (tmp.getLength() >= inputLength) {
+ in.buffer.getBytes(in.start, tmp.getBytes(), 0, inputLength);
+ tmp.setLength(inputLength);
+ } else {
+ byte[] tempArray = new byte[in.end - in.start];
+ in.buffer.getBytes(in.start, tempArray, 0, in.end - in.start);
+ tmp.setBytes(tempArray);
+ }
}
<#else>
value.value++;
@@ -132,10 +134,12 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
@Override
public void output() {
<#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
- ${type.runningType}Holder tmp = (${type.runningType}Holder)value.obj;
- out.start = tmp.start;
- out.end = tmp.end;
- out.buffer = tmp.buffer;
+ org.apache.drill.exec.expr.fn.impl.DrillByteArray tmp = (org.apache.drill.exec.expr.fn.impl.DrillByteArray) value.obj;
+ buf = buf.reallocIfNeeded(tmp.getLength());
+ buf.setBytes(0, tmp.getBytes(), 0, tmp.getLength());
+ out.start = 0;
+ out.end = tmp.getLength();
+ out.buffer = buf;
<#else>
out.value = value.value;
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
index 5281c37..c60d4c9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
@@ -98,11 +98,15 @@ class DrillAggFuncHolder extends DrillFuncHolder{
JVar sizeVar = setupBlock.decl(g.getModel().INT, "vectorSize", JExpr.lit(Integer.MAX_VALUE));
JClass mathClass = g.getModel().ref(Math.class);
for (int id = 0; id<workspaceVars.length; id ++) {
- setupBlock.assign(sizeVar,mathClass.staticInvoke("min").arg(sizeVar).arg(g.getWorkspaceVectors().get(workspaceVars[id]).invoke("getValueCapacity")));
+ if (!workspaceVars[id].isInject()) {
+ setupBlock.assign(sizeVar,mathClass.staticInvoke("min").arg(sizeVar).arg(g.getWorkspaceVectors().get(workspaceVars[id]).invoke("getValueCapacity")));
+ }
}
for(int i =0 ; i < workspaceVars.length; i++) {
- setupBlock.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
+ if (!workspaceVars[i].isInject()) {
+ setupBlock.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
+ }
}
//Use for loop to initialize entries in the workspace vectors.
@@ -148,18 +152,23 @@ class DrillAggFuncHolder extends DrillFuncHolder{
JVar[] workspaceJVars = new JVar[workspaceVars.length];
for(int i =0 ; i < workspaceVars.length; i++){
- Preconditions.checkState(Types.isFixedWidthType(workspaceVars[i].majorType), String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have variable length type.", workspaceVars[i].name, registeredNames[0]));
- Preconditions.checkState(workspaceVars[i].majorType.getMode()==DataMode.REQUIRED, String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have null or repeated type.", workspaceVars[i].name, registeredNames[0]));
+ if (workspaceVars[i].isInject() == true) {
+ workspaceJVars[i] = g.declareClassField("work", g.getModel()._ref(workspaceVars[i].type));
+ g.getBlock(BlockType.SETUP).assign(workspaceJVars[i], g.getMappingSet().getIncoming().invoke("getContext").invoke("getManagedBuffer"));
+ } else {
+ Preconditions.checkState(Types.isFixedWidthType(workspaceVars[i].majorType), String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have variable length type.", workspaceVars[i].name, registeredNames[0]));
+ Preconditions.checkState(workspaceVars[i].majorType.getMode()==DataMode.REQUIRED, String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have null or repeated type.", workspaceVars[i].name, registeredNames[0]));
- //workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType), JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
- workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType));
+ //workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType), JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
+ workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType));
- //Declare a workspace vector for the workspace var.
- TypedFieldId typedFieldId = new TypedFieldId(workspaceVars[i].majorType, g.getWorkspaceTypes().size());
- JVar vv = g.declareVectorValueSetupAndMember(g.getMappingSet().getWorkspace(), typedFieldId);
+ //Declare a workspace vector for the workspace var.
+ TypedFieldId typedFieldId = new TypedFieldId(workspaceVars[i].majorType, g.getWorkspaceTypes().size());
+ JVar vv = g.declareVectorValueSetupAndMember(g.getMappingSet().getWorkspace(), typedFieldId);
- g.getWorkspaceTypes().add(typedFieldId);
- g.getWorkspaceVectors().put(workspaceVars[i], vv);
+ g.getWorkspaceTypes().add(typedFieldId);
+ g.getWorkspaceVectors().put(workspaceVars[i], vv);
+ }
}
return workspaceJVars;
}
@@ -200,6 +209,11 @@ class DrillAggFuncHolder extends DrillFuncHolder{
JVar[] internalVars = new JVar[workspaceJVars.length];
for(int i =0; i < workspaceJVars.length; i++){
+
+ if (workspaceVars[i].isInject()) {
+ internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
+ continue;
+ }
//sub.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
//Access workspaceVar through workspace vector.
JInvocation getValueAccessor = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getAccessor").invoke("get");
@@ -219,6 +233,10 @@ class DrillAggFuncHolder extends DrillFuncHolder{
for(int i =0; i < workspaceJVars.length; i++){
sub.assign(workspaceJVars[i], internalVars[i]);
+ // Injected buffers are not stored as vectors skip storing them in vectors
+ if (workspaceVars[i].isInject()) {
+ continue;
+ }
//Change workspaceVar through workspace vector.
JInvocation setMeth;
if (Types.usesHolderForGet(workspaceVars[i].majorType)) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
index be1d9e6..b524139 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
@@ -21,6 +21,7 @@ package org.apache.drill.exec.expr.fn.impl;
import com.google.common.primitives.UnsignedLongs;
import io.netty.util.internal.PlatformDependent;
+import org.apache.drill.exec.util.DecimalUtility;
public class ByteFunctionHelpers {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
@@ -88,9 +89,73 @@ public class ByteFunctionHelpers {
if (lLen == rLen) return 0;
- return lLen > rLen ? 1 : 0;
+ return lLen > rLen ? 1 : -1;
}
+ public static final int compare(final long laddr, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
+ int lLen = lEnd - lStart;
+ int rLen = rEnd - rStart;
+ int n = Math.min(rLen, lLen);
+ long lPos = laddr + lStart;
+ int rPos = rStart;
+
+
+
+ while (n-- != 0) {
+ byte leftByte = PlatformDependent.getByte(lPos);
+ byte rightByte = right[rPos];
+ if (leftByte != rightByte) {
+ return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+ }
+ lPos++;
+ rPos++;
+ }
+
+ if (lLen == rLen) return 0;
+
+ return lLen > rLen ? 1 : -1;
+
+ }
+ // Get the big endian integer
+ public static int getInteger(byte[] b, int index) {
+ int startIndex = index * DecimalUtility.integerSize;
+
+ if (index == 0) {
+ return (b[startIndex + 3] & 0xFF) |
+ (b[startIndex + 2] & 0xFF) << 8 |
+ (b[startIndex + 1] & 0xFF) << 16 |
+ (b[startIndex] & 0x7F) << 24;
+ }
+
+ return ((b[startIndex + 3] & 0xFF) |
+ (b[startIndex + 2] & 0xFF) << 8 |
+ (b[startIndex + 1] & 0xFF) << 16 |
+ (b[startIndex] & 0xFF) << 24);
+
+ }
+
+ // Set the big endian bytes for the input integer
+ public static void setInteger(byte[] b, int index, int value) {
+ int startIndex = index * DecimalUtility.integerSize;
+ b[startIndex] = (byte) ((value >> 24) & 0xFF);
+ b[startIndex + 1] = (byte) ((value >> 16) & 0xFF);
+ b[startIndex + 2] = (byte) ((value >> 8) & 0xFF);
+ b[startIndex + 3] = (byte) ((value) & 0xFF);
+ }
+
+ // Set the sign in a sparse decimal representation
+ public static void setSign(byte[] b, boolean sign) {
+ int value = getInteger(b, 0);
+ if (sign == true) {
+ setInteger(b, 0, value | 0x80000000);
+ } else {
+ setInteger(b, 0, value & 0x7FFFFFFF);
+ }
+ }
+ // Get the sign
+ public static boolean getSign(byte[] b) {
+ return ((b[0] & 0x80) > 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
new file mode 100644
index 0000000..507db67
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl;
+
+import io.netty.buffer.ByteBuf;
+
+public class DrillByteArray {
+ private byte[] bytes;
+ private int length;
+
+ public DrillByteArray() {
+ this.bytes = new byte[0];
+ this.length = 0;
+ }
+
+ public DrillByteArray(byte[] bytes, int length) {
+ this.bytes = bytes;
+ this.length = length;
+ }
+
+ public DrillByteArray(byte[] bytes) {
+ this(bytes, bytes.length);
+ }
+
+ public void setLength(int length) {
+ this.length = length;
+ }
+
+ public byte[] getBytes() {
+ return this.bytes;
+ }
+
+ public int getLength() {
+ return this.length;
+ }
+
+ public void setBytes(byte[] bytes) {
+ setBytes(bytes, bytes.length);
+ }
+
+ public void setBytes(byte[] bytes, int length) {
+ this.bytes = bytes;
+ this.length = length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
index d429a28..b514adb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
@@ -33,7 +33,7 @@ public class StringFunctionHelpers {
public static void initCap(int start, int end, DrillBuf inBuf, DrillBuf outBuf) {
boolean capNext = true;
int out = 0;
- for (int id = start; id < end; id++; out++) {
+ for (int id = start; id < end; id++, out++) {
byte currentByte = inBuf.getByte(id);
// 'A - Z' : 0x41 - 0x5A
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
index 8e752eb..3d0a9de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
@@ -25,6 +25,8 @@ import java.util.Arrays;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.util.CoreDecimalUtility;
+import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
public class DecimalUtility extends CoreDecimalUtility{
@@ -688,5 +690,34 @@ public class DecimalUtility extends CoreDecimalUtility{
int index = nDecimalDigits - roundUp(scale);
return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1));
}
+
+ public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) {
+ // check the sign first
+ boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
+ boolean rSign = ByteFunctionHelpers.getSign(right);
+ int cmp = 0;
+
+ if (lSign != rSign) {
+ return (lSign == false) ? 1 : -1;
+ }
+
+ // invert the comparison if we are comparing negative numbers
+ int invert = (lSign == true) ? -1 : 1;
+
+ // compare byte by byte
+ int n = 0;
+ int lPos = lStart;
+ int rPos = 0;
+ while (n < length/4) {
+ int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
+ int rightInt = ByteFunctionHelpers.getInteger(right, n);
+ if (leftInt != rightInt) {
+ cmp = (leftInt - rightInt ) > 0 ? 1 : -1;
+ break;
+ }
+ n++;
+ }
+ return cmp * invert;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a99aecad/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index 5aee392..0e40d11 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -539,6 +539,7 @@ public class TestFunctionsQuery {
}
@Test
+ @Ignore ("toString on varchar is not supported. date trunc functions seem to be using that")
public void testDateTrunc() throws Exception {
String query = "select "
+ "date_trunc('MINUTE', time '2:30:21.5') as TIME1, "
[35/47] git commit: DRILL-1021: Windows build
Posted by ja...@apache.org.
DRILL-1021: Windows build
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d944918f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d944918f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d944918f
Branch: refs/heads/master
Commit: d944918ff8a1f76d9b4c400512a130bff7fb2c5f
Parents: 5a7feb9
Author: Parth Chandra <pc...@maprtech.com>
Authored: Wed Jun 11 18:10:34 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 13:33:48 2014 -0700
----------------------------------------------------------------------
contrib/native/client/CMakeLists.txt | 25 ++-
.../client/cmakeModules/FindZookeeper.cmake | 28 ++-
.../client/patches/zookeeper-3.4.6-x64.patch | 163 +++++++++++++++++
contrib/native/client/readme.win.txt | 179 +++++++++++++++++++
.../native/client/src/clientlib/CMakeLists.txt | 5 +
.../native/client/src/clientlib/drillClient.cpp | 1 +
.../client/src/clientlib/drillClientImpl.cpp | 6 +
.../client/src/clientlib/drillClientImpl.hpp | 5 +
.../native/client/src/include/drill/common.hpp | 8 +
.../native/client/src/protobuf/CMakeLists.txt | 4 +
pom.xml | 2 +-
11 files changed, 418 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/contrib/native/client/CMakeLists.txt b/contrib/native/client/CMakeLists.txt
index 31ac472..603586d 100644
--- a/contrib/native/client/CMakeLists.txt
+++ b/contrib/native/client/CMakeLists.txt
@@ -26,9 +26,16 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmakeModules/")
# Find Boost
-set(Boost_USE_STATIC_LIBS OFF)
-set(Boost_USE_MULTITHREADED ON)
-set(Boost_USE_STATIC_RUNTIME OFF)
+if(MSVC)
+ set(Boost_USE_STATIC_LIBS ON)
+ set(Boost_USE_MULTITHREADED ON)
+ set(Boost_USE_STATIC_RUNTIME OFF)
+else()
+ set(Boost_USE_STATIC_LIBS OFF)
+ set(Boost_USE_MULTITHREADED ON)
+ set(Boost_USE_STATIC_RUNTIME OFF)
+endif()
+
find_package(Boost 1.53.0 REQUIRED COMPONENTS regex system date_time chrono thread )
include_directories(${Boost_INCLUDE_DIRS})
@@ -36,8 +43,18 @@ if(CMAKE_COMPILER_IS_GNUCXX)
set(CMAKE_EXE_LINKER_FLAGS "-lrt -lpthread")
set(CMAKE_CXX_FLAGS "-fPIC")
endif()
+if(MSVC)
+ set(CMAKE_CXX_FLAGS "/EHsc")
+endif()
-add_definitions(-DBOOST_ALL_DYN_LINK)
+if(MSVC)
+ # ask boost to not to auto link any libraries
+ add_definitions(-DBOOST_ALL_NO_LIB)
+ # use static libs with zookeeper
+ add_definitions(-DUSE_STATIC_LIB)
+else()
+ add_definitions(-DBOOST_ALL_DYN_LINK)
+endif()
# Find Protobufs
find_package(Protobuf REQUIRED )
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/cmakeModules/FindZookeeper.cmake
----------------------------------------------------------------------
diff --git a/contrib/native/client/cmakeModules/FindZookeeper.cmake b/contrib/native/client/cmakeModules/FindZookeeper.cmake
index c727ad2..fd8247f 100644
--- a/contrib/native/client/cmakeModules/FindZookeeper.cmake
+++ b/contrib/native/client/cmakeModules/FindZookeeper.cmake
@@ -27,10 +27,32 @@
#pkg_check_modules(PC_LIBXML QUIET libxml-2.0)
#set(Zookeeper_DEFINITIONS ${PC_LIBXML_CFLAGS_OTHER})
-find_path(Zookeeper_INCLUDE_DIR zookeeper/zookeeper.h /usr/local/include)
+if (MSVC)
+ if(${CMAKE_BUILD_TYPE} MATCHES "Debug")
+ set(ZK_BuildOutputDir "Debug")
+ else()
+ set(ZK_BuildOutputDir "Release")
+ endif()
+ if("${ZOOKEEPER_HOME}_" MATCHES "^_$")
+ message(" ")
+ message("- Please set the cache variable ZOOKEEPER_HOME to point to the directory with the zookeeper source.")
+ message("- CMAKE will look for zookeeper include files in $ZOOKEEPER_HOME/src/c/include.")
+ message("- CMAKE will look for zookeeper library files in $ZOOKEEPER_HOME/src/c/Debug or $ZOOKEEPER_HOME/src/c/Release.")
+ else()
+ FILE(TO_CMAKE_PATH ${ZOOKEEPER_HOME} Zookeeper_HomePath)
+ set(Zookeeper_LIB_PATHS ${Zookeeper_HomePath}/src/c/${ZK_BuildOutputDir})
+
+ find_path(ZK_INCLUDE_DIR zookeeper.h ${Zookeeper_HomePath}/src/c/include)
+ find_path(ZK_INCLUDE_DIR_GEN zookeeper.jute.h ${Zookeeper_HomePath}/src/c/generated)
+ set(Zookeeper_INCLUDE_DIR zookeeper.h ${ZK_INCLUDE_DIR} ${ZK_INCLUDE_DIR_GEN} )
+ find_library(Zookeeper_LIBRARY NAMES zookeeper PATHS ${Zookeeper_LIB_PATHS})
+ endif()
+else()
+ set(Zookeeper_LIB_PATHS /usr/local/lib /opt/local/lib)
+ find_path(Zookeeper_INCLUDE_DIR zookeeper/zookeeper.h /usr/local/include)
+ find_library(Zookeeper_LIBRARY NAMES zookeeper_mt PATHS ${Zookeeper_LIB_PATHS})
+endif()
-set(Zookeeper_LIB_PATHS /usr/local/lib /opt/local/lib)
-find_library(Zookeeper_LIBRARY NAMES zookeeper_mt PATHS ${Zookeeper_LIB_PATHS})
set(Zookeeper_LIBRARIES ${Zookeeper_LIBRARY} )
set(Zookeeper_INCLUDE_DIRS ${Zookeeper_INCLUDE_DIR} )
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/patches/zookeeper-3.4.6-x64.patch
----------------------------------------------------------------------
diff --git a/contrib/native/client/patches/zookeeper-3.4.6-x64.patch b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch
new file mode 100644
index 0000000..96f2d10
--- /dev/null
+++ b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch
@@ -0,0 +1,163 @@
+From 64697ddd8a90f29d1693658f04e975e435e3c869 Mon Sep 17 00:00:00 2001
+From: unknown <no...@NorrisL.simba.ad>
+Date: Thu, 5 Jun 2014 16:40:48 -0700
+Subject: [PATCH] Allow zookeeper to build in x64
+
+---
+ src/c/include/winstdint.h | 4 ++++
+ src/c/src/mt_adaptor.c | 54 +++++++++++++++++++++++------------------------
+ 2 files changed, 30 insertions(+), 28 deletions(-)
+
+diff --git a/src/c/include/winstdint.h b/src/c/include/winstdint.h
+index d02608a..df405f7 100644
+--- a/src/c/include/winstdint.h
++++ b/src/c/include/winstdint.h
+@@ -40,6 +40,9 @@
+ #pragma once
+ #endif
+
++#if (_MSC_VER > 1500) // Visual Studio 2010 and Beyond
++#include <stdint.h>
++#else
+ #include <limits.h>
+
+ // For Visual Studio 6 in C++ mode and for many Visual Studio versions when
+@@ -244,4 +247,5 @@ typedef uint64_t uintmax_t;
+ #endif // __STDC_CONSTANT_MACROS ]
+
+
++#endif
+ #endif // _MSC_STDINT_H_ ]
+diff --git a/src/c/src/mt_adaptor.c b/src/c/src/mt_adaptor.c
+index 974063f..5ce0fd9 100644
+--- a/src/c/src/mt_adaptor.c
++++ b/src/c/src/mt_adaptor.c
+@@ -114,7 +114,7 @@ int process_async(int outstanding_sync)
+ unsigned __stdcall do_io( void * );
+ unsigned __stdcall do_completion( void * );
+
+-int handle_error(SOCKET sock, char* message)
++int handle_error(zhandle_t* zh, SOCKET sock, char* message)
+ {
+ LOG_ERROR(("%s. %d",message, WSAGetLastError()));
+ closesocket (sock);
+@@ -122,7 +122,7 @@ int handle_error(SOCKET sock, char* message)
+ }
+
+ //--create socket pair for interupting selects.
+-int create_socket_pair(SOCKET fds[2])
++int create_socket_pair(zhandle_t* zh, SOCKET fds[2])
+ {
+ struct sockaddr_in inaddr;
+ struct sockaddr addr;
+@@ -141,23 +141,23 @@ int create_socket_pair(SOCKET fds[2])
+ inaddr.sin_port = 0; //--system assigns the port
+
+ if ( setsockopt(lst,SOL_SOCKET,SO_REUSEADDR,(char*)&yes,sizeof(yes)) == SOCKET_ERROR ) {
+- return handle_error(lst,"Error trying to set socket option.");
++ return handle_error(zh, lst,"Error trying to set socket option.");
+ }
+ if (bind(lst,(struct sockaddr *)&inaddr,sizeof(inaddr)) == SOCKET_ERROR){
+- return handle_error(lst,"Error trying to bind socket.");
++ return handle_error(zh, lst,"Error trying to bind socket.");
+ }
+ if (listen(lst,1) == SOCKET_ERROR){
+- return handle_error(lst,"Error trying to listen on socket.");
++ return handle_error(zh, lst,"Error trying to listen on socket.");
+ }
+ len=sizeof(inaddr);
+ getsockname(lst, &addr,&len);
+ fds[0]=socket(AF_INET, SOCK_STREAM,0);
+ if (connect(fds[0],&addr,len) == SOCKET_ERROR){
+- return handle_error(lst, "Error while connecting to socket.");
++ return handle_error(zh, lst, "Error while connecting to socket.");
+ }
+ if ((fds[1]=accept(lst,0,0)) == INVALID_SOCKET){
+ closesocket(fds[0]);
+- return handle_error(lst, "Error while accepting socket connection.");
++ return handle_error(zh, lst, "Error while accepting socket connection.");
+ }
+ closesocket(lst);
+ return 0;
+@@ -238,11 +238,11 @@ int adaptor_init(zhandle_t *zh)
+
+ /* We use a pipe for interrupting select() in unix/sol and socketpair in windows. */
+ #ifdef WIN32
+- if (create_socket_pair(adaptor_threads->self_pipe) == -1){
++ if (create_socket_pair(zh, adaptor_threads->self_pipe) == -1){
+ LOG_ERROR(("Can't make a socket."));
+ #else
+ if(pipe(adaptor_threads->self_pipe)==-1) {
+- LOG_ERROR(("Can't make a pipe %d",errno));
++ LOG_ERROR(LOGCALLBACK(zh), "Can't make a pipe %d",errno);
+ #endif
+ free(adaptor_threads);
+ return -1;
+@@ -255,6 +255,7 @@ int adaptor_init(zhandle_t *zh)
+ zh->adaptor_priv = adaptor_threads;
+ pthread_mutex_init(&zh->to_process.lock,0);
+ pthread_mutex_init(&adaptor_threads->zh_lock,0);
++ pthread_mutex_init(&adaptor_threads->zh_lock,0);
+ // to_send must be recursive mutex
+ pthread_mutexattr_init(&recursive_mx_attr);
+ pthread_mutexattr_settype(&recursive_mx_attr, PTHREAD_MUTEX_RECURSIVE);
+@@ -364,7 +365,7 @@ void *do_io(void *v)
+
+ api_prolog(zh);
+ notify_thread_ready(zh);
+- LOG_DEBUG(("started IO thread"));
++ LOG_DEBUG(LOGCALLBACK(zh), "started IO thread");
+ fds[0].fd=adaptor_threads->self_pipe[0];
+ fds[0].events=POLLIN;
+ while(!zh->close_requested) {
+@@ -483,25 +484,9 @@ int32_t inc_ref_counter(zhandle_t* zh,int i)
+ int32_t fetch_and_add(volatile int32_t* operand, int incr)
+ {
+ #ifndef WIN32
+- int32_t result;
+- asm __volatile__(
+- "lock xaddl %0,%1\n"
+- : "=r"(result), "=m"(*(int *)operand)
+- : "0"(incr)
+- : "memory");
+- return result;
++ return __sync_fetch_and_add(operand, incr);
+ #else
+- volatile int32_t result;
+- _asm
+- {
+- mov eax, operand; //eax = v;
+- mov ebx, incr; // ebx = i;
+- mov ecx, 0x0; // ecx = 0;
+- lock xadd dword ptr [eax], ecx;
+- lock xadd dword ptr [eax], ebx;
+- mov result, ecx; // result = ebx;
+- }
+- return result;
++ return InterlockedExchangeAdd(operand, incr);
+ #endif
+ }
+
+@@ -515,6 +500,19 @@ __attribute__((constructor)) int32_t get_xid()
+ return fetch_and_add(&xid,1);
+ }
+
++void lock_reconfig(struct _zhandle *zh)
++{
++ struct adaptor_threads *adaptor = zh->adaptor_priv;
++ if(adaptor)
++ pthread_mutex_lock(&adaptor->zh_lock);
++}
++void unlock_reconfig(struct _zhandle *zh)
++{
++ struct adaptor_threads *adaptor = zh->adaptor_priv;
++ if(adaptor)
++ pthread_mutex_lock(&adaptor->zh_lock);
++}
++
+ void enter_critical(zhandle_t* zh)
+ {
+ struct adaptor_threads *adaptor = zh->adaptor_priv;
+--
+1.9.2.msysgit.0
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/readme.win.txt
----------------------------------------------------------------------
diff --git a/contrib/native/client/readme.win.txt b/contrib/native/client/readme.win.txt
new file mode 100644
index 0000000..df6ee64
--- /dev/null
+++ b/contrib/native/client/readme.win.txt
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Building the Drill C++ Client on Windows
+
+This document lists the steps to build the Drill C++ Client on Windows. The
+steps and examples are for Windows 7, using Visual Studio 2010 Express. Newer
+Windows platforms should be more or less similar.
+
+1 Tools and software
+
+1.1 The following git tools would be useful to get source code and apply patches -
+ MSysGit and GitForWindows - http://msysgit.github.io/
+ Tortoise SVN - http://tortoisesvn.net/
+
+1.2 Windows SDK
+ The Windows SDK is required to enable 64 bit options for Visual Studio 2010
+ Express. You can get it here:
+ http://www.microsoft.com/en-us/download/details.aspx?id=8279
+ Note: For 64 bit builds, change the platform toolset to Windows SDK for your
+ project.
+ (Root node, i.e. the projectname) Properties ->Configuration Properties->General->Platform Toolset = Windows7.1SDK
+
+1.3 [Optional] Windows Driver kit
+ The Windows Driver kit is required to get the 64 bit assembler ml64. The
+ 32 bit assembler masm is included in VS2010. The assembler is required to build
+ boost and can be ignored for Drill
+
+ ML64(the 64 bit assembler) can be downloaded from here:
+ http://www.microsoft.com/en-us/download/details.aspx?id=11800
+ MASM(the 32 bit assembler) should be installed in VC/bin already. If not it
+ can be downloaded from here:
+ http://www.microsoft.com/en-us/download/details.aspx?id=12654
+
+ Add the paths to the assemblers in your path environment variable. ml64 can be
+ found in (or the appropriate path to the WinDDK installation on your system):
+ C:\WinDDK\7600.16385.1\bin\x86\amd64
+
+2 Dependencies
+
+2.0
+ a) The Drill client library requires the following third party libraries -
+ boost
+ zookeeper C API
+ protobufs
+ The Drill client is linked with STATIC versions of these libraries. The
+ libraries are themselves linked with the DYNAMIC C Runtime DLLs. It is
+ important that the libraries all have the same linkage model, otherwise the
+ Drill Client library will not link successfully.
+ b) The build assumes that Zookeeper is not availble as a
+ binary installer and is available in source form only for windows. The
+ location of the header files, as a result is different from Unix like
+ systems. This will be important later with Cmake.
+ c) The document below will refer to the following as the directories where
+ the thirdparty library source is installed.
+ BOOST_HOME - Directory where boost source is installed.
+ ZOOKEEPER_HOME - Directory where Zookeeper source is installed. Note that
+ this is the directory for the full Zookeeper source not just the
+ source for the C library.
+ PROTOBUF_HOME - Directory where Protobuf source is installed.
+
+2.1 Boost (version 1.55)
+ a) Download Boost from:
+ http://www.boost.org/users/history/version_1_55_0.html
+ b) i) Boost 32 bit build -
+ Open a Visual Studio command prompt from the Visual Studio IDE
+ ii) Boost 64 bit build -
+ Open a command prompt from the Windows SDK menu
+ Start->All Programs->Windows SDK 7.1->Start Command Prompt
+ c) In the command prompt window -
+ C:> cd <BOOST_HOME>
+ C:> .\bootstrap.bat
+ d) Choose the build type (64 bit, 32 bit) and the variant type (debug, release)
+ and build the libraries. Boost build will write the libraries to
+ <BOOST_HOME>/stage/lib. Copy them to an appropriately named directory
+
+ C:> .\b2 variant=debug link=static threading=multi address-model=64 toolset=msvc runtime-link=shared
+ C:> mkdir Debug64
+ C:> copy stage\lib\* Debug64
+
+ C:> .\b2 variant=release link=static threading=multi address-model=64 toolset=msvc runtime-link=shared
+ C:> mkdir Release64
+ C:> copy stage\lib\* Release64
+
+ C:> .\b2 variant=debug link=static threading=multi address-model=32 toolset=msvc runtime-link=shared
+ C:> mkdir Debug32
+ C:> copy stage\lib\* Debug32
+
+ C:> .\b2 variant=release link=static threading=multi address-model=32 toolset=msvc runtime-link=shared
+ C:> mkdir Release32
+ C:> copy stage\lib\* Release32
+ e) Notes:
+ i) For more information on Boost build
+ http://www.boost.org/doc/libs/1_55_0/more/getting_started/windows.html#the-boost-distribution
+ ii) Detail options for b2 -
+ http://www.boost.org/boost-build2/doc/html/bbv2/overview/invocation.html
+ iii) If you do not have the 64 bit assembler installed, boost-context does not
+ build. It is safe to ignore it as boost-context is not needed for Drill
+
+2.2 Protobuf (2.5.0)
+ a) Protobuf builds static libraries
+ b) In Visual Studio, open <PROTOBUF_HOME>/vsprojects/protobuf.sln. The IDE may
+ update the solution file. This should go thru successfully.
+ c) If build for 64 bit, add a 64 bit project configuration for each project. (Make sure the
+ platform toolset is set to Windows7.1SDK)
+ d) Build
+
+2.3 Zookeeper (3.4.6)
+ a) Set the ZOOKEEPER_HOME environment variable
+ b) The 3.4.6 release of Zookeeper does not build correctly on 64 bit windows. To
+ fix that for the 64 bit build, apply patch zookeeper-3.4.6-x64.patch
+ For example in Msysgit
+ $ cd <ZOOKEEPER_HOME> && git apply <DRILL_HOME>/contrib/native/client/patches/zookeeper-3.4.6-x64.patch
+ c) InVisual Studio 2010 Express open <ZOOKEEPER_HOME>/src/c/zookeeper.sln
+ i) Add a 64 bit project configuration for each project. (Make sure the
+ platform toolset is set to Windows7.1SDK)
+ ii) Change the output type for the zookeeper project to a static lib
+ Properties->Configuration Properties->General->Configuration Type = Static Library
+ iii) In the cli project add the preprocessor define USE_STATIC_LIB
+ iiii) Build. Build zookeeper lib first, then build cli
+
+3 Building Drill Clientlib
+3.1 SET the following environment variables
+ set BOOST_LIBRARYDIR=<BOOST_HOME>\BUILD_TYPE
+ set BOOST_INCLUDEDIR=<BOOST_HOME>
+
+3.2 Generate the Visual Studio Solutions file
+ C:> cd <DRILL_HOME>/contrib/native/client
+ C:> mkdir build
+ C:> cd build
+
+ a) For the 32 bit build :
+ C:> cmake -G "Visual Studio 10" -D ZOOKEEPER_HOME=<ZOOKEPER_HOME> -D PROTOBUF_SRC_ROOT_FOLDER=<PROTOBUF_HOME> -D CMAKE_BUILD_TYPE=Debug ..
+
+ b) For the 64 bit build :
+ C:> cmake -G "Visual Studio 10 Win64 " -D ZOOKEEPER_HOME=<ZOOKEPER_HOME> -D PROTOBUF_SRC_ROOT_FOLDER=<PROTOBUF_HOME> -D CMAKE_BUILD_TYPE=Debug ..
+
+3.3 Open the generated <DRILL_HOME>/contrib/native/client/build/drillclient.sln
+ file in Visual Studio.
+ a) If doing a Debug build, check the link libraries for the clientlib
+ project. You might see the protobuf library being linked from the Release
+ build. Correct that to pick up the library from the Debug build. (See 4.2
+ below)
+
+3.4 Select the ALL_BUILD project and build.
+
+4 Common Problems
+4.1 In the 64 bit build, If you get: " error LNK1112: module machine type 'x64' conflicts with target machine type 'X86'"
+ a) Some library is a 32 bit library and you're linking it into a 64 bit Drill
+ Client library, or vice versa. You can check the type of the libraryi with
+ dumpbin.
+ dumpbin library_or_dll /headers
+ b) Check the linker settings for the project.
+ Properties->Configuration Properties->Linker->Command Line->Additional Options.
+ If you're doing a 64 bit build and you see "/machine:X86", remove it and rebuild
+
+4.2) error LNK2038: mismatch detected for '_ITERATOR_DEBUG_LEVEL': value '0' doesn't match value '2' in decimalUtils.obj
+ a) This happens if you are doing a Debug build and linking a Release build
+ version of some library, or vice versa. Check the link libraries.
+ Properties->Configuration Properties->Linker->Input->Additional Dependencies.
+ Check the libraries are all the same as your configuration (all debug, or all
+ release).
+ In particular, for debug builds, check the path of the protobuf library.
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/src/clientlib/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/CMakeLists.txt b/contrib/native/client/src/clientlib/CMakeLists.txt
index dc8f032..a2e7052 100644
--- a/contrib/native/client/src/clientlib/CMakeLists.txt
+++ b/contrib/native/client/src/clientlib/CMakeLists.txt
@@ -41,5 +41,10 @@ set_property(
PROPERTY COMPILE_DEFINITIONS_DEBUG DEBUG DEBUG=1 THREADED
)
+if(MSVC)
+ set(CMAKE_CXX_FLAGS "/EHsc")
+ add_definitions(-DDRILL_CLIENT_EXPORTS)
+endif()
+
add_library(drillClient SHARED ${CLIENTLIB_SRC_FILES} )
target_link_libraries(drillClient ${Boost_LIBRARIES} ${PROTOBUF_LIBRARY} ${Zookeeper_LIBRARIES} protomsgs y2038)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/src/clientlib/drillClient.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp
index 6611332..a7aafaa 100644
--- a/contrib/native/client/src/clientlib/drillClient.cpp
+++ b/contrib/native/client/src/clientlib/drillClient.cpp
@@ -17,6 +17,7 @@
*/
+#include "drill/common.hpp"
#include "drill/drillClient.hpp"
#include "drill/recordBatch.hpp"
#include "drillClientImpl.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/src/clientlib/drillClientImpl.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp
index 0ea4897..f1a165d 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.cpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+
+#include "drill/common.hpp"
#include <queue>
#include <string.h>
#include <boost/asio.hpp>
@@ -23,7 +25,11 @@
#include <boost/date_time/posix_time/posix_time_duration.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/thread.hpp>
+#ifdef _WIN32
+#include <zookeeper.h>
+#else
#include <zookeeper/zookeeper.h>
+#endif
#include "drill/drillClient.hpp"
#include "drill/recordBatch.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/src/clientlib/drillClientImpl.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp
index 3ac0b20..5ac158f 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.hpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp
@@ -28,6 +28,7 @@
//#define BOOST_ASIO_DISABLE_IOCP
//#endif // _WIN32
+#include "drill/common.hpp"
#include <stdlib.h>
#include <time.h>
#include <queue>
@@ -35,7 +36,11 @@
#include <boost/asio.hpp>
#include <boost/asio/deadline_timer.hpp>
#include <boost/thread.hpp>
+#ifdef _WIN32
+#include <zookeeper.h>
+#else
#include <zookeeper/zookeeper.h>
+#endif
#include "drill/common.hpp"
#include "drill/drillClient.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/src/include/drill/common.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/include/drill/common.hpp b/contrib/native/client/src/include/drill/common.hpp
index 59734dc..dfb04e8 100644
--- a/contrib/native/client/src/include/drill/common.hpp
+++ b/contrib/native/client/src/include/drill/common.hpp
@@ -20,6 +20,14 @@
#ifndef _COMMON_H_
#define _COMMON_H_
+#ifdef _WIN32
+// The order of inclusion is important. Including winsock2 before everything else
+// ensures that the correct typedefs are defined and that the older typedefs defined
+// in winsock and windows.h are not picked up.
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
#include <stdint.h>
#include <string>
#include <vector>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/contrib/native/client/src/protobuf/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/protobuf/CMakeLists.txt b/contrib/native/client/src/protobuf/CMakeLists.txt
index 154138d..1d0b3af 100644
--- a/contrib/native/client/src/protobuf/CMakeLists.txt
+++ b/contrib/native/client/src/protobuf/CMakeLists.txt
@@ -103,6 +103,10 @@ add_custom_target(cpProtobufs
#message("ProtoHeaders = ${ProtoHeaders}" )
#message("ProtoIncludes = ${ProtoIncludes}" )
+if(MSVC)
+ set(CMAKE_CXX_FLAGS "/EHsc")
+endif()
+
add_library(protomsgs STATIC ${ProtoSources} ${ProtoHeaders} ${ProtoIncludes} )
#set linker properties. The first time around, the protobufs generated files may not exist
# and CMAKE will not be able to determine the linker type.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d944918f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index de02c92..b46fde8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,7 +181,7 @@
<exclude>**/*.project</exclude>
<exclude>**/*.checkstyle</exclude>
<exclude>.*/**</exclude>
- <exclude>*.patch</exclude>
+ <exclude>**/*.patch</exclude>
<exclude>**/*.pb.cc</exclude>
<exclude>**/*.pb.h</exclude>
<exclude>**/*.linux</exclude>
[07/47] git commit: Pass correct uncompressed data size to BytesInput
ctor in ComplexParquetReader
Posted by ja...@apache.org.
Pass correct uncompressed data size to BytesInput ctor in ComplexParquetReader
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4216e0e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4216e0e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4216e0e2
Branch: refs/heads/master
Commit: 4216e0e2c60cf17caa678cee685cbfc2ca4e819a
Parents: ed72c13
Author: Parth Chandra <pc...@maprtech.com>
Authored: Sat Aug 23 21:48:04 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 08:28:34 2014 -0700
----------------------------------------------------------------------
.../src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4216e0e2/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java b/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
index 379d3e6..516be0e 100644
--- a/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
+++ b/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
@@ -147,7 +147,7 @@ public class ColumnChunkIncReadStore implements PageReadStore {
ByteBuffer buffer = buf.nioBuffer(0, pageHeader.compressed_page_size);
CompatibilityUtil.getBuf(in, buffer, pageHeader.compressed_page_size);
return new Page(
- decompressor.decompress(BytesInput.from(buffer, 0, pageHeader.compressed_page_size), pageHeader.compressed_page_size),
+ decompressor.decompress(BytesInput.from(buffer, 0, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
pageHeader.data_page_header.num_values,
pageHeader.uncompressed_page_size,
parquetMetadataConverter.fromParquetStatistics(pageHeader.data_page_header.statistics, columnDescriptor.getType()),
[24/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
index df60650..12f45b2 100644
--- a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
@@ -53,27 +53,7 @@ outside:{
</#if>
<#if mode == "var">
-
- for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
- byte leftByte = left.buffer.getByte(l);
- byte rightByte = right.buffer.getByte(r);
- if (leftByte != rightByte) {
- ${output} = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
- break outside;
- }
- }
-
- int l = (left.end - left.start) - (right.end - right.start);
- if (l > 0) {
- ${output} = 1;
- break outside;
- } else if (l == 0) {
- ${output} = 0;
- break outside;
- } else {
- ${output} = -1;
- break outside;
- }
+ ${output} = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end);
<#elseif mode == "fixed">
${output} = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);
</#if>
@@ -99,8 +79,11 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
public class GCompare${left}${right}{
@@ -219,27 +202,8 @@ public class GCompare${left}${right}{
public void setup(RecordBatch b) {}
public void eval() {
-
<#if type.mode == "var" >
-outside:
- {
- if (left.end - left.start == right.end - right.start) {
- int n = left.end - left.start;
- int l = left.start;
- int r = right.start;
- while (n-- !=0) {
- byte leftByte = left.buffer.getByte(l++);
- byte rightByte = right.buffer.getByte(r++);
- if (leftByte != rightByte) {
- out.value = 0;
- break outside;
- }
- }
- out.value = 1;
- } else {
- out.value = 0;
- }
- }
+ out.value = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end);
<#else>
out.value = left.value == right.value ? 1 : 0;
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java b/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
index 0d2be65..89a400d 100644
--- a/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
+++ b/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java
@@ -49,15 +49,15 @@ public class ${className} implements DrillSimpleFunc {
<#elseif (minor.class == "Interval")>
output.months = input.months;
output.days = input.days;
- output.milliSeconds = input.milliSeconds;
+ output.milliseconds = input.milliseconds;
<#elseif (minor.class == "IntervalDay")>
output.days = input.days;
- output.milliSeconds = input.milliSeconds;
+ output.milliseconds = input.milliseconds;
<#elseif minor.class.startsWith("Decimal")>
output.scale = input.scale;
output.precision = input.precision;
<#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
- output.setSign(input.getSign());
+ output.setSign(input.getSign(input.start, input.buffer), output.start, output.buffer);
output.start = input.start;
output.buffer = input.buffer;
<#else>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalAggrFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalAggrFunctions1.java b/exec/java-exec/src/main/codegen/templates/DateIntervalAggrFunctions1.java
index 701e992..1c9b76f 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalAggrFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalAggrFunctions1.java
@@ -60,10 +60,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#if type.runningType == "Interval">
value.months = ${type.initialValue};
value.days= ${type.initialValue};
- value.milliSeconds = ${type.initialValue};
+ value.milliseconds = ${type.initialValue};
<#elseif type.runningType == "IntervalDay">
value.days= ${type.initialValue};
- value.milliSeconds = ${type.initialValue};
+ value.milliseconds = ${type.initialValue};
<#else>
value.value = ${type.initialValue};
</#if>
@@ -90,13 +90,13 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
long inMS = (long) in.months * org.apache.drill.exec.expr.fn.impl.DateUtility.monthsToMillis+
in.days * (org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis) +
- in.milliSeconds;
+ in.milliseconds;
value.value = Math.min(value.value, inMS);
<#elseif type.outputType == "IntervalDay">
long inMS = (long) in.days * (org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis) +
- in.milliSeconds;
+ in.milliseconds;
value.value = Math.min(value.value, inMS);
@@ -113,12 +113,12 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#elseif type.outputType == "Interval">
long inMS = (long) in.months * org.apache.drill.exec.expr.fn.impl.DateUtility.monthsToMillis+
in.days * (org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis) +
- in.milliSeconds;
+ in.milliseconds;
value.value = Math.max(value.value, inMS);
<#elseif type.outputType == "IntervalDay">
long inMS = (long) in.days * (org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis) +
- in.milliSeconds;
+ in.milliseconds;
value.value = Math.max(value.value, inMS);
<#else>
@@ -129,10 +129,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#if type.outputType == "Interval">
value.days += in.days;
value.months += in.months;
- value.milliSeconds += in.milliSeconds;
+ value.milliseconds += in.milliseconds;
<#elseif type.outputType == "IntervalDay">
value.days += in.days;
- value.milliSeconds += in.milliSeconds;
+ value.milliseconds += in.milliseconds;
<#else>
value.value += in.value;
</#if>
@@ -154,10 +154,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
out.months = (int) (value.value / org.apache.drill.exec.expr.fn.impl.DateUtility.monthsToMillis);
value.value = value.value % org.apache.drill.exec.expr.fn.impl.DateUtility.monthsToMillis;
out.days = (int) (value.value / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
- out.milliSeconds = (int) (value.value % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (value.value % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
<#elseif type.outputType == "IntervalDay">
out.days = (int) (value.value / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
- out.milliSeconds = (int) (value.value % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (value.value % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
<#else>
out.value = value.value;
</#if>
@@ -165,10 +165,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#if type.outputType == "Interval">
out.months = value.months;
out.days = value.days;
- out.milliSeconds = value.milliSeconds;
+ out.milliseconds = value.milliseconds;
<#elseif type.outputType == "IntervalDay">
out.days = value.days;
- out.milliSeconds = value.milliSeconds;
+ out.milliseconds = value.milliseconds;
<#else>
out.value = value.value;
</#if>
@@ -180,10 +180,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#if type.runningType == "Interval">
value.months = ${type.initialValue};
value.days= ${type.initialValue};
- value.milliSeconds = ${type.initialValue};
+ value.milliseconds = ${type.initialValue};
<#elseif type.runningType == "IntervalDay">
value.days= ${type.initialValue};
- value.milliSeconds = ${type.initialValue};
+ value.milliseconds = ${type.initialValue};
<#else>
value.value = ${type.initialValue};
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
index 0074c7c..e119958 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
@@ -55,12 +55,12 @@ public static class G${type}Difference implements DrillSimpleFunc {
public void eval() {
<#if type == "Time">
- out.milliSeconds = left.value - right.value;
+ out.milliseconds = left.value - right.value;
<#elseif type == "Date">
out.days = (int) ((left.value - right.value) / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
<#elseif type == "TimeStamp">
long difference = (left.value - right.value);
- out.milliSeconds = (int) (difference % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (difference % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
out.days = (int) (difference / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
<#elseif type == "TimeStampTZ">
if (left.index != right.index) {
@@ -70,7 +70,7 @@ public static class G${type}Difference implements DrillSimpleFunc {
left.value = (leftInput.withZone(org.joda.time.DateTimeZone.forID(org.apache.drill.exec.expr.fn.impl.DateUtility.timezoneList[right.index]))).getMillis();
}
long difference = (left.value - right.value);
- out.milliSeconds = (int) (difference % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (difference % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
out.days = (int) (difference / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
</#if>
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
index 6fc78eb..349b7bf 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
@@ -63,12 +63,12 @@ public class ${datetype}${intervaltype}Functions {
<#if intervaltype == "Interval">
${temp}.addMonths(${right}.months <#if op == '-'> * -1 </#if>);
${temp}.addDays(${right}.days <#if op == '-'> * -1 </#if>);
- ${temp}.add(${right}.milliSeconds <#if op == '-'> * -1 </#if>);
+ ${temp}.add(${right}.milliseconds <#if op == '-'> * -1 </#if>);
<#elseif intervaltype == "IntervalYear">
${temp}.addMonths(${right}.value <#if op == '-'> * -1 </#if>);
<#elseif intervaltype == "IntervalDay">
${temp}.addDays(${right}.days <#if op == '-'> * -1 </#if>);
- ${temp}.add(${right}.milliSeconds <#if op == '-'> * -1 </#if>);
+ ${temp}.add(${right}.milliseconds <#if op == '-'> * -1 </#if>);
<#elseif intervaltype == "Int" || intervaltype == "BigInt">
${temp}.addDays((int) ${right}.value <#if op == '-'> * -1 </#if>);
</#if>
@@ -184,7 +184,7 @@ public class ${datetype}${intervaltype}Functions {
// Needn't add anything to time from interval year data type. Output is same as input
${output} = ${left}.value;
<#else>
- ${output} = ${left}.value ${op} ${right}.milliSeconds;
+ ${output} = ${left}.value ${op} ${right}.milliseconds;
// Wrap around 24 hour clock if we exceeded it while adding the time component
${output} = ${output} % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
index 326b1df..695cfcf 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
@@ -28,6 +28,8 @@ import org.apache.drill.exec.expr.annotations.Workspace;
package org.apache.drill.exec.expr.fn.impl;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -38,6 +40,7 @@ import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
@SuppressWarnings("unused")
@FunctionTemplate(name = "to_char", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
@@ -45,14 +48,14 @@ public class G${type}ToChar implements DrillSimpleFunc {
@Param ${type}Holder left;
@Param VarCharHolder right;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Workspace org.joda.time.MutableDateTime temp;
@Workspace org.joda.time.format.DateTimeFormatter format;
@Output VarCharHolder out;
public void setup(RecordBatch b) {
temp = new org.joda.time.MutableDateTime(0, org.joda.time.DateTimeZone.UTC);
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[100]);
+ buffer = buffer.reallocIfNeeded(100);
// Get the desired output format and create a DateTimeFormatter
byte[] buf = new byte[right.end - right.start];
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
index 6dae417..0858c3f 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/Extract.java
@@ -98,12 +98,12 @@ public class ${className} {
<#elseif toUnit == "Day">
out.value = in.days;
<#elseif toUnit == "Hour">
- out.value = in.milliSeconds/(org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+ out.value = in.milliseconds/(org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
<#elseif toUnit == "Minute">
- int millis = in.milliSeconds % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+ int millis = in.milliseconds % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
out.value = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
<#elseif toUnit == "Second">
- long millis = in.milliSeconds % org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis;
+ long millis = in.milliseconds % org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis;
out.value = (double) millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
</#if>
<#elseif fromUnit == "IntervalDay">
@@ -112,12 +112,12 @@ public class ${className} {
<#elseif toUnit == "Day">
out.value = in.days;
<#elseif toUnit == "Hour">
- out.value = in.milliSeconds/(org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+ out.value = in.milliseconds/(org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
<#elseif toUnit == "Minute">
- int millis = in.milliSeconds % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+ int millis = in.milliseconds % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
out.value = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
<#elseif toUnit == "Second">
- long millis = in.milliSeconds % org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis;
+ long millis = in.milliseconds % org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis;
out.value = (double) millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
</#if>
<#else> <#-- IntervalYear type -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java
index ad769e1..484c8f4 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalIntervalArithmetic.java
@@ -58,12 +58,12 @@ public class ${intervaltype}Functions {
<#if intervaltype == "Interval">
out.months = left.months + right.months;
out.days = left.days + right.days;
- out.milliSeconds = left.milliSeconds + right.milliSeconds;
+ out.milliseconds = left.milliseconds + right.milliseconds;
<#elseif intervaltype == "IntervalYear">
out.value = left.value + right.value;
<#elseif intervaltype == "IntervalDay">
out.days = left.days + right.days;
- out.milliSeconds = left.milliSeconds + right.milliSeconds;
+ out.milliseconds = left.milliseconds + right.milliseconds;
</#if>
}
}
@@ -82,12 +82,12 @@ public class ${intervaltype}Functions {
<#if intervaltype == "Interval">
out.months = left.months - right.months;
out.days = left.days - right.days;
- out.milliSeconds = left.milliSeconds - right.milliSeconds;
+ out.milliseconds = left.milliseconds - right.milliseconds;
<#elseif intervaltype == "IntervalYear">
out.value = left.value - right.value;
<#elseif intervaltype == "IntervalDay">
out.days = left.days - right.days;
- out.milliSeconds = left.milliSeconds - right.milliSeconds;
+ out.milliseconds = left.milliseconds - right.milliseconds;
</#if>
}
}
@@ -104,12 +104,12 @@ public class ${intervaltype}Functions {
<#if intervaltype == "Interval">
out.months = -left.months;
out.days = -left.days;
- out.milliSeconds = -left.milliSeconds;
+ out.milliseconds = -left.milliseconds;
<#elseif intervaltype == "IntervalYear">
out.value = -left.value;
<#elseif intervaltype == "IntervalDay">
out.days = -left.days;
- out.milliSeconds = -left.milliSeconds;
+ out.milliseconds = -left.milliseconds;
</#if>
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java
index 9a56c1b..ce73f74 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/IntervalNumericArithmetic.java
@@ -49,12 +49,12 @@ public class ${intervaltype}${numerictype}Functions {
<#if intervaltype == "Interval">
${out}.months = (int) (${left}.months * ${right}.value);
${out}.days = (int) (${left}.days * ${right}.value);
- ${out}.milliSeconds = (int) (${left}.milliSeconds * ${right}.value);
+ ${out}.milliseconds = (int) (${left}.milliseconds * ${right}.value);
<#elseif intervaltype == "IntervalYear">
${out}.months = (int) (${left}.value * ${right}.value);
<#elseif intervaltype == "IntervalDay">
${out}.days = (int) (${left}.days * ${right}.value);
- ${out}.milliSeconds = (int) (${left}.milliSeconds * ${right}.value);
+ ${out}.milliseconds = (int) (${left}.milliseconds * ${right}.value);
</#if>
</#macro>
@@ -67,12 +67,12 @@ public class ${intervaltype}${numerictype}Functions {
<#if intervaltype == "Interval">
fractionalMonths = (${left}.months ${op} (double) ${right}.value);
fractionalDays = (${left}.days ${op} (double) ${right}.value);
- fractionalMillis = (${left}.milliSeconds ${op} (double) ${right}.value);
+ fractionalMillis = (${left}.milliseconds ${op} (double) ${right}.value);
<#elseif intervaltype == "IntervalYear">
fractionalMonths = (${left}.value ${op} (double) ${right}.value);
<#elseif intervaltype == "IntervalDay">
fractionalDays = (${left}.days ${op} ${right}.value);
- fractionalMillis = (${left}.milliSeconds ${op} (double) ${right}.value);
+ fractionalMillis = (${left}.milliseconds ${op} (double) ${right}.value);
</#if>
${out}.months = (int) fractionalMonths;
@@ -86,7 +86,7 @@ public class ${intervaltype}${numerictype}Functions {
fractionalDays = fractionalDays - (long) fractionalDays;
fractionalMillis += fractionalDays * org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
- ${out}.milliSeconds = (int) fractionalMillis;
+ ${out}.milliseconds = (int) fractionalMillis;
</#macro>
@SuppressWarnings("unused")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToTimeStampFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToTimeStampFunction.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToTimeStampFunction.java
index 9195891..4fd619f 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToTimeStampFunction.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToTimeStampFunction.java
@@ -64,7 +64,7 @@ public class G${numerics}ToTimeStamp implements DrillSimpleFunc {
value = value.multiply(millisConstant);
inputMillis = (new java.math.BigDecimal(value, left.scale)).longValue();
<#elseif (numerics == "Decimal28Sparse") || (numerics == "Decimal38Sparse")>
- java.math.BigDecimal input = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(left.buffer, left.start, left.nDecimalDigits, left.scale);
+ java.math.BigDecimal input = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(left.buffer, left.start, left.nDecimalDigits, left.scale);
inputMillis = input.multiply(new java.math.BigDecimal(1000)).longValue();
</#if>
<#else>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/DateIntervalFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctions.java
index b518c44..8fe13bb 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctions.java
@@ -213,9 +213,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalCompareBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds" output="out.value"/>
+ <@intervalCompareBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds" output="out.value"/>
<#else>
- <@intervalCompareBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds" output="out.value"/>
+ <@intervalCompareBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds" output="out.value"/>
</#if>
}
@@ -234,9 +234,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds"/>
<#else>
- <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds"/>
</#if>
out.value = leftMS < rightMS ? 1 : 0;
@@ -256,9 +256,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds"/>
<#else>
- <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds"/>
</#if>
out.value = leftMS <= rightMS ? 1 : 0;
@@ -278,9 +278,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds"/>
<#else>
- <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds"/>
</#if>
out.value = leftMS > rightMS ? 1 : 0;
@@ -300,9 +300,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds"/>
<#else>
- <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds"/>
</#if>
out.value = leftMS >= rightMS ? 1 : 0;
@@ -321,9 +321,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds"/>
<#else>
- <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds"/>
</#if>
out.value = leftMS == rightMS ? 1 : 0;
@@ -342,9 +342,9 @@ public class GCompare${type.name}Functions {
public void eval() {
<#if type.name == "Interval">
- <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="left.months" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="right.months" rightDays="right.days" rightMillis="right.milliseconds"/>
<#else>
- <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliSeconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliSeconds"/>
+ <@intervalConvertBlock left="left" right="right" leftMonths="0" leftDays="left.days" leftMillis="left.milliseconds" rightMonths="0" rightDays="right.days" rightMillis="right.milliseconds"/>
</#if>
out.value = leftMS != rightMS ? 1 : 0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
index e9c03c8..cb0dc58 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
@@ -26,6 +26,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -34,7 +36,10 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -42,15 +47,14 @@ import java.nio.ByteBuffer;
public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Param ${type.from}Holder in;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = (${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
@@ -59,11 +63,11 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
// Re initialize the buffer everytime
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
out.scale = (int) scale.value;
out.precision = (int) precision.value;
- out.setSign(in.getSign());
+ out.setSign(in.getSign(in.start, in.buffer), out.start, out.buffer);
/* We store base 1 Billion integers in our representation, which requires
* 30 bits, but a typical integer requires 32 bits. In our dense representation
@@ -71,7 +75,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
* representation rearrange the bits so that we use 32 bits represent the digits.
*/
- byte[] intermediateBytes = new byte[(in.nDecimalDigits * org.apache.drill.common.util.DecimalUtility.integerSize) + 1];
+ byte[] intermediateBytes = new byte[(in.nDecimalDigits * org.apache.drill.exec.util.DecimalUtility.integerSize) + 1];
int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
@@ -100,7 +104,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
inputIndex++;
intermediateIndex++;
- if (((intermediateIndex - 1) % org.apache.drill.common.util.DecimalUtility.integerSize) == 0) {
+ if (((intermediateIndex - 1) % org.apache.drill.exec.util.DecimalUtility.integerSize) == 0) {
shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
maskIndex++;
shiftOrder -= 2;
@@ -115,7 +119,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
* Now we transfer the bytes into a integer array and separate out the scale and
* integer part of the decimal. Also pad the scale part with zeroes if needed
*/
- int[] intermediate = new int[(intermediateBytes.length/org.apache.drill.common.util.DecimalUtility.integerSize) + 1];
+ int[] intermediate = new int[(intermediateBytes.length/org.apache.drill.exec.util.DecimalUtility.integerSize) + 1];
java.nio.ByteBuffer wrapper = java.nio.ByteBuffer.wrap(intermediateBytes);
intermediate[0] = wrapper.get(0);
@@ -132,27 +136,27 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
int dstIndex = out.nDecimalDigits - 1;
// break the scale and integer part and pad zeroes
- if (in.scale > 0 && (actualDigits = (in.scale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS)) > 0) {
+ if (in.scale > 0 && (actualDigits = (in.scale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS)) > 0) {
- int paddedDigits = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - actualDigits;
+ int paddedDigits = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - actualDigits;
int padding = (int) (Math.pow(10, paddedDigits));
int transferDigitMask = (int) (Math.pow(10, actualDigits));
/* copy the remaining scale over to the last deciml digit */
- out.setInteger(dstIndex, ((intermediate[srcIndex] % transferDigitMask) * (padding)));
+ out.setInteger(dstIndex, ((intermediate[srcIndex] % transferDigitMask) * (padding)), out.start, out.buffer);
dstIndex--;
while (srcIndex > 0) {
- out.setInteger(dstIndex, ((intermediate[srcIndex]/transferDigitMask) + ((intermediate[srcIndex - 1] % transferDigitMask) * padding)));
+ out.setInteger(dstIndex, ((intermediate[srcIndex]/transferDigitMask) + ((intermediate[srcIndex - 1] % transferDigitMask) * padding)), out.start, out.buffer);
dstIndex--;
srcIndex--;
}
- out.setInteger(dstIndex, (intermediate[0]/transferDigitMask));
+ out.setInteger(dstIndex, (intermediate[0]/transferDigitMask), out.start, out.buffer);
} else {
for (; srcIndex >= 0; srcIndex--, dstIndex--)
- out.setInteger(dstIndex, intermediate[srcIndex]);
+ out.setInteger(dstIndex, intermediate[srcIndex], out.start, out.buffer);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
index 92cd576..4d8d1a5 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
@@ -34,7 +34,7 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
import io.netty.buffer.ByteBuf;
import java.nio.ByteBuffer;
@@ -52,7 +52,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void eval() {
// Divide the decimal with the scale to get the floating point value
- out.value = ((${type.javatype}) (in.value)) / (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale));
+ out.value = ((${type.javatype}) (in.value)) / (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) in.scale));
}
}
<#elseif type.major == "DecimalComplexFloat" || type.major == "DecimalComplexDouble"> <#-- Cast function template for conversion from Decimal9, Decimal18 to Float4 -->
@@ -63,6 +63,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -70,7 +72,7 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
import io.netty.buffer.ByteBuf;
import java.nio.ByteBuffer;
@@ -88,9 +90,9 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void eval() {
<#if type.from == "Decimal28Dense" || type.from == "Decimal38Dense">
- java.math.BigDecimal bigDecimal = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(in.buffer, in.start, in.nDecimalDigits, in.scale, in.maxPrecision, in.WIDTH);
+ java.math.BigDecimal bigDecimal = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(in.buffer, in.start, in.nDecimalDigits, in.scale, in.maxPrecision, in.WIDTH);
<#else>
- java.math.BigDecimal bigDecimal = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(in.buffer, in.start, in.nDecimalDigits, in.scale, true);
+ java.math.BigDecimal bigDecimal = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(in.buffer, in.start, in.nDecimalDigits, in.scale, true);
</#if>
out.value = bigDecimal.${type.javatype}Value();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java
index 57c38a1..c435f6d 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java
@@ -28,6 +28,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -35,7 +37,7 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
import io.netty.buffer.ByteBuf;
import java.nio.ByteBuffer;
@@ -52,9 +54,9 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void eval() {
- int carry = (org.apache.drill.common.util.DecimalUtility.getFirstFractionalDigit(in.value, in.scale) > 4) ? 1 : 0;
+ int carry = (org.apache.drill.exec.util.DecimalUtility.getFirstFractionalDigit(in.value, in.scale) > 4) ? 1 : 0;
// Assign the integer part of the decimal to the output holder
- out.value = java.lang.Math.abs((${type.javatype}) (org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(in.value, (int) in.scale))) + carry;
+ out.value = java.lang.Math.abs((${type.javatype}) (org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(in.value, (int) in.scale))) + carry;
if (in.value < 0) {
out.value *= -1;
}
@@ -75,7 +77,7 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
import io.netty.buffer.ByteBuf;
import java.nio.ByteBuffer;
@@ -92,19 +94,19 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void eval() {
- int carry = (org.apache.drill.common.util.DecimalUtility.getFirstFractionalDigit(in.buffer, in.scale, in.start, in.nDecimalDigits) > 4) ? 1 : 0;
+ int carry = (org.apache.drill.exec.util.DecimalUtility.getFirstFractionalDigit(in.buffer, in.scale, in.start, in.nDecimalDigits) > 4) ? 1 : 0;
// Get the index, where the integer part of the decimal ends
- int integerEndIndex = in.nDecimalDigits - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale);
+ int integerEndIndex = in.nDecimalDigits - org.apache.drill.exec.util.DecimalUtility.roundUp(in.scale);
for (int i = 0 ; i < integerEndIndex; i++) {
// We store values as base 1 billion integers, use this to compute the output (we don't care about overflows)
- out.value = (${type.javatype}) ((out.value * org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) + in.getInteger(i));
+ out.value = (${type.javatype}) ((out.value * org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) + in.getInteger(i, in.start, in.buffer));
}
out.value += carry;
- if (in.getSign() == true) {
+ if (in.getSign(in.start, in.buffer) == true) {
out.value *= -1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
index 9a2d3e5..8c46b49 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
@@ -28,6 +28,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -36,7 +38,10 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -44,15 +49,14 @@ import java.nio.ByteBuffer;
public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Param ${type.from}Holder in;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = (${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
@@ -61,25 +65,25 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
out.start = 0;
out.scale = (int) scale.value;
out.precision = (int) precision.value;
- boolean sign = (in.getSign());
+ boolean sign = (in.getSign(in.start, in.buffer));
// Re initialize the buffer everytime
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
int inputIdx = in.nDecimalDigits - 1;
int outputIdx = out.nDecimalDigits - 1;
for (; inputIdx >= 0; inputIdx--, outputIdx--) {
- out.setInteger(outputIdx, in.getInteger(inputIdx));
+ out.setInteger(outputIdx, in.getInteger(inputIdx, in.start, in.buffer), out.start, out.buffer);
}
// round up or down the scale
if (in.scale != out.scale) {
- org.apache.drill.common.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
+ org.apache.drill.exec.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
}
- out.setSign(sign);
+ out.setSign(sign, in.start, in.buffer);
}
}
</#if> <#-- type.major -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
index e9e9711..0f8790f 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
@@ -26,6 +26,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -34,7 +36,10 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -42,15 +47,14 @@ import java.nio.ByteBuffer;
public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Param ${type.from}Holder in;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = (${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
@@ -60,7 +64,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
// Re initialize the buffer everytime
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
out.scale = (int) scale.value;
@@ -78,9 +82,9 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
int index = in.nDecimalDigits - 1;
int actualDigits;
- if (in.scale > 0 && (actualDigits = (in.scale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS)) > 0) {
+ if (in.scale > 0 && (actualDigits = (in.scale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS)) > 0) {
- int paddedDigits = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - actualDigits;
+ int paddedDigits = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - actualDigits;
int paddedMask = (int) Math.pow(10, paddedDigits);
@@ -88,16 +92,16 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
* digit, so we have padded zeroes to it for ease of arithmetic
* Truncate the extra zeroes added and move the digits to the right
*/
- int temp = (in.getInteger(index)/paddedMask);
+ int temp = (in.getInteger(index, in.start, in.buffer)/paddedMask);
index--;
while(index >= 0) {
- int transferDigits = (in.getInteger(index) % (paddedMask));
+ int transferDigits = (in.getInteger(index, in.start, in.buffer) % (paddedMask));
intermediate[index] = (int) (temp + (Math.pow(10, actualDigits) * transferDigits));
- temp = (in.getInteger(index)/(paddedMask));
+ temp = (in.getInteger(index, in.start, in.buffer)/(paddedMask));
index--;
}
@@ -108,7 +112,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
* need only 5 decimal digit to be stored, simply copy over the integers
*/
for (int i = 1; i < in.nDecimalDigits; i++)
- intermediate[i - 1] = in.getInteger(i);
+ intermediate[i - 1] = in.getInteger(i, in.start, in.buffer);
}
@@ -120,14 +124,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
*/
/* Allocate a byte array */
- int size = (((intermediate.length - 1) * org.apache.drill.common.util.DecimalUtility.integerSize) + 1);
+ int size = (((intermediate.length - 1) * org.apache.drill.exec.util.DecimalUtility.integerSize) + 1);
byte[] intermediateBytes = new byte[size];
java.nio.ByteBuffer wrapper = java.nio.ByteBuffer.wrap(intermediateBytes);
wrapper.put((byte) intermediate[0]);
for (int i = 1; i < intermediate.length; i++) {
- wrapper.put(java.nio.ByteBuffer.allocate(org.apache.drill.common.util.DecimalUtility.integerSize).putInt(intermediate[i]).array());
+ wrapper.put(java.nio.ByteBuffer.allocate(org.apache.drill.exec.util.DecimalUtility.integerSize).putInt(intermediate[i]).array());
}
final int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
@@ -135,21 +139,21 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
int shiftOrder = 2;
// Start just after the last integer and shift bits to the right
- index = size - (org.apache.drill.common.util.DecimalUtility.integerSize+ 1);
+ index = size - (org.apache.drill.exec.util.DecimalUtility.integerSize+ 1);
while (index >= 0) {
/* get the last bits that need to shifted to the next byte */
byte shiftBits = (byte) ((intermediateBytes[index] & mask[maskIndex]) << (8 - shiftOrder));
- int shiftOrder1 = ((index % org.apache.drill.common.util.DecimalUtility.integerSize) == 0) ? shiftOrder - 2 : shiftOrder;
+ int shiftOrder1 = ((index % org.apache.drill.exec.util.DecimalUtility.integerSize) == 0) ? shiftOrder - 2 : shiftOrder;
/* transfer the bits from the left to the right */
intermediateBytes[index + 1] = (byte) (((intermediateBytes[index + 1] & 0xFF) >>> (shiftOrder1)) | shiftBits);
index--;
- if ((index % org.apache.drill.common.util.DecimalUtility.integerSize) == 0) {
+ if ((index % org.apache.drill.exec.util.DecimalUtility.integerSize) == 0) {
/* We are on a border */
shiftOrder += 2;
maskIndex++;
@@ -167,7 +171,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
// Set the bytes in the buffer
out.buffer.setBytes(dstIndex, intermediateBytes, 1, (size - 1));
- out.setSign(in.getSign());
+ out.setSign(in.getSign(in.start, in.buffer), out.start, out.buffer);
}
}
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java
index dc8eca5..ed62444 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java
@@ -28,6 +28,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -35,10 +37,13 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.SwappedByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -47,12 +52,11 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferSize}]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ buffer = buffer.reallocIfNeeded(${type.bufferSize});
}
public void eval() {
@@ -67,7 +71,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
in.value *= -1;
}
- ${type.javatype} separator = (${type.javatype}) org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale);
+ ${type.javatype} separator = (${type.javatype}) org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) in.scale);
str.append(in.value / separator);
@@ -90,7 +94,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
*
* We missed the initial zeroes in the fractional part. Below logic accounts for this case
*/
- str.append(org.apache.drill.common.util.DecimalUtility.toStringWithZeroes((in.value % separator), in.scale));
+ str.append(org.apache.drill.exec.util.DecimalUtility.toStringWithZeroes((in.value % separator), in.scale));
}
out.buffer = buffer;
@@ -107,6 +111,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -114,9 +120,11 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -125,28 +133,27 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder len;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferSize}]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ buffer = buffer.reallocIfNeeded((int) len.value);
}
public void eval() {
StringBuilder str = new StringBuilder();
int index = 0;
- int fractionalStartIndex = ${type.arraySize} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale);
+ int fractionalStartIndex = ${type.arraySize} - org.apache.drill.exec.util.DecimalUtility.roundUp(in.scale);
// Find the first non-zero value in the integer part of the decimal
- while (index < fractionalStartIndex && in.getInteger(index) == 0) {
+ while (index < fractionalStartIndex && in.getInteger(index, in.start, in.buffer) == 0) {
index++;
}
// If we have valid digits print '-' sign
- if ((in.getSign() == true) && index < ${type.arraySize}) {
+ if ((in.getSign(in.start, in.buffer) == true) && index < ${type.arraySize}) {
str.append("-");
}
@@ -159,10 +166,10 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// convert the integer part
while (index < fractionalStartIndex) {
- int value = in.getInteger(index++);
+ int value = in.getInteger(index++, in.start, in.buffer);
if (fillZeroes == true) {
- str.append(org.apache.drill.common.util.DecimalUtility.toStringWithZeroes(value, org.apache.drill.common.util.DecimalUtility.MAX_DIGITS));
+ str.append(org.apache.drill.exec.util.DecimalUtility.toStringWithZeroes(value, org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS));
} else {
str.append(value);
fillZeroes = true;
@@ -178,25 +185,25 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
* as it might have padding that needs to be stripped
*/
while (fractionalStartIndex < ${type.arraySize} - 1) {
- int value = in.getInteger(fractionalStartIndex++);
+ int value = in.getInteger(fractionalStartIndex++, in.start, in.buffer);
// Fill zeroes at the beginning of the decimal digit
- str.append(org.apache.drill.common.util.DecimalUtility.toStringWithZeroes(value, org.apache.drill.common.util.DecimalUtility.MAX_DIGITS));
+ str.append(org.apache.drill.exec.util.DecimalUtility.toStringWithZeroes(value, org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS));
}
// Last decimal digit, strip the extra zeroes we may have padded
- int actualDigits = in.scale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS;
+ int actualDigits = in.scale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS;
- int lastFractionalDigit = in.getInteger(${type.arraySize} - 1);
+ int lastFractionalDigit = in.getInteger(${type.arraySize} - 1, in.start, in.buffer);
if (actualDigits != 0) {
// Strip padded zeroes at the end that is not part of the scale
- lastFractionalDigit /= (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - actualDigits)));
- str.append(org.apache.drill.common.util.DecimalUtility.toStringWithZeroes(lastFractionalDigit, actualDigits));
+ lastFractionalDigit /= (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - actualDigits)));
+ str.append(org.apache.drill.exec.util.DecimalUtility.toStringWithZeroes(lastFractionalDigit, actualDigits));
} else {
// Last digit does not have any padding print as is
- str.append(org.apache.drill.common.util.DecimalUtility.toStringWithZeroes(lastFractionalDigit, org.apache.drill.common.util.DecimalUtility.MAX_DIGITS));
+ str.append(org.apache.drill.exec.util.DecimalUtility.toStringWithZeroes(lastFractionalDigit, org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
index 516ec82..bd86819 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
@@ -26,6 +26,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -33,9 +35,12 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -56,12 +61,12 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
out.value = (int) in.value;
// Check if we need to truncate or round up
if (out.scale > in.scale) {
- out.value *= (int) org.apache.drill.common.util.DecimalUtility.getPowerOfTen(out.scale - in.scale);
+ out.value *= (int) org.apache.drill.exec.util.DecimalUtility.getPowerOfTen(out.scale - in.scale);
} else if (out.scale < in.scale) {
// need to round up since we are truncating fractional part
- int scaleFactor = (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale));
- int newScaleFactor = (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) scale.value));
- int truncScaleFactor = (int) (org.apache.drill.common.util.DecimalUtility.getPowerOfTen( (int) (Math.abs(in.scale - scale.value))));
+ int scaleFactor = (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) in.scale));
+ int newScaleFactor = (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen((int) scale.value));
+ int truncScaleFactor = (int) (org.apache.drill.exec.util.DecimalUtility.getPowerOfTen( (int) (Math.abs(in.scale - scale.value))));
int truncFactor = (int) (in.scale - scale.value);
// Assign the integer part
@@ -71,11 +76,11 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
int fractionalPart = (int) (in.value % scaleFactor);
// From the entire fractional part extract the digits upto which rounding is needed
- int newFractionalPart = (int) (org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(fractionalPart, truncFactor));
+ int newFractionalPart = (int) (org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(fractionalPart, truncFactor));
int truncatedFraction = fractionalPart % truncScaleFactor;
// Get the truncated fractional part and extract the first digit to see if we need to add 1
- int digit = Math.abs((int) org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(truncatedFraction, truncFactor - 1));
+ int digit = Math.abs((int) org.apache.drill.exec.util.DecimalUtility.adjustScaleDivide(truncatedFraction, truncFactor - 1));
if (digit > 4) {
if (in.value > 0) {
@@ -95,6 +100,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -102,9 +109,11 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -120,7 +129,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
}
public void eval() {
- java.math.BigDecimal temp = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
+ java.math.BigDecimal temp = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
temp = temp.setScale((int) scale.value, java.math.BigDecimal.ROUND_HALF_UP);
out.value = temp.unscaledValue().${type.javatype}Value();
out.precision = (int) precision.value;
@@ -134,6 +143,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -141,9 +152,11 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -153,23 +166,22 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
@Param BigIntHolder precision;
@Param BigIntHolder scale;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Output ${type.to}Holder out;
public void setup(RecordBatch incoming) {
- int size = (${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize));
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ buffer = buffer.reallocIfNeeded(size);
}
public void eval() {
- java.math.BigDecimal temp = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
+ java.math.BigDecimal temp = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer, in.start, in.nDecimalDigits, in.scale);
temp = temp.setScale((int) scale.value, java.math.BigDecimal.ROUND_HALF_UP);
out.precision = (int) precision.value;
out.scale = (int) scale.value;
out.buffer = buffer;
out.start = 0;
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(temp, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(temp, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
}
}
</#if> <#-- type.major -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
index f4a5c77..c05b2e5 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
@@ -29,6 +29,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -36,9 +38,12 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -47,7 +52,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
<#if type.major == "FloatDecimalComplex" || type.major == "DoubleDecimalComplex">
-@Workspace ByteBuf buffer;
+@Inject DrillBuf buffer;
</#if>
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@@ -55,9 +60,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void setup(RecordBatch incoming) {
<#if type.major == "FloatDecimalComplex" || type.major == "DoubleDecimalComplex">
- int size = ${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize);
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
+ buffer = buffer.reallocIfNeeded(size);
</#if>
}
@@ -71,12 +75,12 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
out.buffer = buffer;
// Assign the integer part of the decimal to the output holder
- org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
+ org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
<#elseif type.to.endsWith("Decimal9")>
- out.value = org.apache.drill.common.util.DecimalUtility.getDecimal9FromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.scale, out.precision);
+ out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal9FromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.scale, out.precision);
<#elseif type.to.endsWith("Decimal18")>
- out.value = org.apache.drill.common.util.DecimalUtility.getDecimal18FromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.scale, out.precision);
+ out.value = org.apache.drill.exec.util.DecimalUtility.getDecimal18FromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.scale, out.precision);
</#if>
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
index 78bde2e..139fff8 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
@@ -25,6 +25,8 @@
package org.apache.drill.exec.expr.fn.impl.gcast;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -32,9 +34,12 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.Workspace;
+
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.nio.ByteBuffer;
@SuppressWarnings("unused")
@@ -43,7 +48,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Param ${type.from}Holder in;
<#if type.to.startsWith("Decimal28") || type.to.startsWith("Decimal38")>
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
</#if>
@Param BigIntHolder precision;
@Param BigIntHolder scale;
@@ -51,9 +56,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void setup(RecordBatch incoming) {
<#if type.to.startsWith("Decimal28") || type.to.startsWith("Decimal38")>
- int size = ${type.arraySize} * (org.apache.drill.common.util.DecimalUtility.integerSize);
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[size]);
- buffer = new io.netty.buffer.SwappedByteBuf(buffer);
+ int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
+ buffer = buffer.reallocIfNeeded(size);
</#if>
}
@@ -66,7 +70,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
out.value = (${type.javatype}) in.value;
// converting from integer to decimal, pad zeroes if scale is non zero
- out.value = (${type.javatype}) org.apache.drill.common.util.DecimalUtility.adjustScaleMultiply(out.value, (int) scale.value);
+ out.value = (${type.javatype}) org.apache.drill.exec.util.DecimalUtility.adjustScaleMultiply(out.value, (int) scale.value);
<#else>
out.start = 0;
@@ -74,21 +78,21 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// Initialize the buffer
for (int i = 0; i < ${type.arraySize}; i++) {
- out.setInteger(i, 0);
+ out.setInteger(i, 0, out.start, out.buffer);
}
// check if input is a negative number and store the sign
if (in.value < 0) {
- out.setSign(true);
+ out.setSign(true, out.start, out.buffer);
}
// Figure out how many array positions to be left for the scale part
- int scaleSize = org.apache.drill.common.util.DecimalUtility.roundUp((int) scale.value);
+ int scaleSize = org.apache.drill.exec.util.DecimalUtility.roundUp((int) scale.value);
int integerIndex = (${type.arraySize} - scaleSize - 1);
while (in.value != 0 && integerIndex >= 0) {
- out.setInteger(integerIndex--, (int) Math.abs((in.value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE)));
- in.value = in.value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE;
+ out.setInteger(integerIndex--, (int) Math.abs((in.value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE)), out.start, out.buffer);
+ in.value = in.value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
}
</#if>
[16/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 73b7b53..457e0b1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -30,7 +30,9 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.Float4Holder;
import org.apache.drill.exec.expr.holders.Float8Holder;
@@ -85,6 +87,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -132,6 +135,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -177,6 +181,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -223,6 +228,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -269,6 +275,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -287,7 +294,7 @@ public class TestCastFunctions extends PopUnitTestBase{
for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
VarCharHolder holder0 = new VarCharHolder();
a0.get(i, holder0);
- assertEquals("123", holder0.toString());
+ assertEquals("123", StringFunctionHelpers.toStringFromUTF8(holder0.start, holder0.end, holder0.buffer));
++count;
}
@@ -316,6 +323,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -334,7 +342,7 @@ public class TestCastFunctions extends PopUnitTestBase{
for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
VarBinaryHolder holder0 = new VarBinaryHolder();
a0.get(i, holder0);
- assertEquals("123", holder0.toString());
+ assertEquals("123", StringFunctionHelpers.toStringFromUTF8(holder0.start, holder0.end, holder0.buffer));
++count;
}
@@ -361,6 +369,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -408,6 +417,7 @@ public class TestCastFunctions extends PopUnitTestBase{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = allocator;
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java
index 6cb6ec1..da018c3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java
@@ -26,6 +26,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -62,6 +63,7 @@ public class TestComparisonFunctions extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
String planString = Resources.toString(Resources.getResource(COMPARISON_TEST_PHYSICAL_PLAN), Charsets.UTF_8).replaceAll("EXPRESSION", expression);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
index 0a021ad..5111a49 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.ArrayList;
import java.util.List;
@@ -36,7 +36,6 @@ import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.util.ByteBufUtil;
import org.apache.drill.exec.util.ByteBufUtil.HadoopWritables;
import org.apache.drill.exec.util.VectorUtil;
import org.apache.drill.exec.vector.ValueVector;
@@ -323,7 +322,7 @@ public class TestConvertFunctions extends BaseTestQuery {
public void testHadooopVInt() throws Exception {
final int _0 = 0;
final int _9 = 9;
- final ByteBuf buffer = ByteBufUtil.createBuffer(_9);
+ final DrillBuf buffer = getAllocator().buffer(_9);
long longVal = 0;
buffer.clear();
@@ -356,6 +355,7 @@ public class TestConvertFunctions extends BaseTestQuery {
HadoopWritables.writeVInt(buffer, _0, _9, Integer.MIN_VALUE);
intVal = HadoopWritables.readVInt(buffer, _0, _9);
assertEquals(intVal, Integer.MIN_VALUE);
+ buffer.release();
}
protected <T> void verifySQL(String sql, T expectedResults) throws Throwable {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java
index 3e4081d..2ff99c8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java
@@ -24,6 +24,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -72,6 +73,7 @@ public class TestImplicitCastFunctions extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index a686fa9..93a387f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 18478a3..d551319 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -38,8 +38,10 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers;
import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -156,6 +158,7 @@ public class TestSimpleFunctions extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -174,7 +177,7 @@ public class TestSimpleFunctions extends ExecTest {
if (!a1.isNull(i)) {
NullableVarCharHolder holder = new NullableVarCharHolder();
a1.get(i, holder);
- assertEquals("aaaa", holder.toString());
+ assertEquals("aaaa", StringFunctionHelpers.toStringFromUTF8(holder.start, holder.end, holder.buffer));
++count;
}
}
@@ -197,6 +200,7 @@ public class TestSimpleFunctions extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -216,7 +220,7 @@ public class TestSimpleFunctions extends ExecTest {
NullableVarCharHolder holder = new NullableVarCharHolder();
a1.get(i, holder);
//when offset is negative, substring return empty string.
- assertEquals("", holder.toString());
+ assertEquals("", StringFunctionHelpers.toStringFromUTF8(holder.start, holder.end, holder.buffer));
++count;
}
}
@@ -239,6 +243,7 @@ public class TestSimpleFunctions extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -257,7 +262,7 @@ public class TestSimpleFunctions extends ExecTest {
if (!a1.isNull(i)) {
NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
a1.get(i, holder);
- assertEquals("aa", holder.toString());
+ assertEquals("aa", StringFunctionHelpers.toStringFromUTF8(holder.start, holder.end, holder.buffer));
++count;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
index 0f91366..2e60893 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
@@ -24,6 +24,8 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.cache.local.LocalCache;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -34,6 +36,8 @@ import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.junit.Test;
@@ -69,12 +73,12 @@ public class TestStringFunctions extends ExecTest {
public void runTest(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection, Object[] expectedResults, String planPath) throws Throwable {
-
new NonStrictExpectations(){{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8);
@@ -91,7 +95,7 @@ public class TestStringFunctions extends ExecTest {
for (int i = 0; i<res.length; i++) {
assertEquals(String.format("column %s does not match", i), expectedResults[i], res[i]);
- }
+ }
}
if(context.getFailureCause() != null){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
index 3669c5d..b39566a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -57,6 +58,7 @@ public class TestAgg extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
index e3b4daf..b02249d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
@@ -22,6 +22,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -51,6 +52,7 @@ private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnec
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
index b95244d..a112d92 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
@@ -25,6 +25,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -60,6 +61,7 @@ public class TestSimpleFilter extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
@@ -89,6 +91,7 @@ public class TestSimpleFilter extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
index b7bd9e8..ed50685 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
@@ -32,6 +32,7 @@ import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.cache.local.LocalCache;
import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -83,6 +84,7 @@ public class TestHashJoin extends PopUnitTestBase{
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
bitContext.getOptionManager(); result = opt;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -267,8 +269,8 @@ public class TestHashJoin extends PopUnitTestBase{
assertEquals(272, count);
}
}
-
-
+
+
@Test
public void testHashJoinExprInCondition() throws Exception {
RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
index 65d51dc..7b264c4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
@@ -29,6 +29,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -77,6 +78,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -128,6 +130,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext));
@@ -182,6 +185,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext));
@@ -236,6 +240,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext));
@@ -286,6 +291,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bitContext.getAllocator(); result = new TopLevelAllocator();;
bitContext.getConfig(); result = c;
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -393,5 +399,5 @@ public class TestMergeJoin extends PopUnitTestBase {
assertEquals(10, count);
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
index 0d5d622..87ee259 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
@@ -28,6 +28,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -62,6 +63,7 @@ public class TestSimpleLimit extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
verifyLimitCount(bitContext, connection, "test1.json", 5);
@@ -74,6 +76,7 @@ public class TestSimpleLimit extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
verifyLimitCount(bitContext, connection, "test3.json", 95);
@@ -91,6 +94,7 @@ public class TestSimpleLimit extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
verifyLimitCount(bitContext, connection, "test2.json", 69999);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
index 63fb54b..514e913 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
@@ -27,6 +27,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -67,6 +68,7 @@ public class TestSimpleProjection extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
index f73889d..4f8ba7e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -64,6 +65,7 @@ public class TestSimpleSort extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
@@ -113,6 +115,7 @@ public class TestSimpleSort extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
index e05a170..bb7ec69 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
@@ -25,6 +25,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -64,6 +65,7 @@ public class TestSVRemover extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
index abe3410..d0d4005 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
@@ -24,6 +24,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -37,6 +38,7 @@ import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.ValueVector;
import org.junit.Test;
import com.codahale.metrics.MetricRegistry;
@@ -65,6 +67,7 @@ public class TestTraceMultiRecordBatch extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
@@ -74,6 +77,9 @@ public class TestTraceMultiRecordBatch extends ExecTest {
SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
while(exec.next()) {
+ for(ValueVector vv: exec){
+ vv.clear();
+ }
}
exec.stop();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
index 5817e0b..f6766b1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.cache.VectorAccessibleSerializable;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -80,6 +81,7 @@ public class TestTraceOutputDump extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
index 576f017..3420169 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
@@ -27,6 +27,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
@@ -62,6 +63,7 @@ public class TestSimpleUnion extends ExecTest {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
index 33bfc71..831bf74 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.record.vector;
import static org.junit.Assert.assertEquals;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.List;
@@ -68,7 +69,7 @@ public class TestLoad extends ExecTest {
for (int i = 0; i < byteBufs.length; i++) {
bytes += byteBufs[i].writerIndex();
}
- ByteBuf byteBuf = allocator.buffer(bytes);
+ DrillBuf byteBuf = allocator.buffer(bytes);
int index = 0;
for (int i = 0; i < byteBufs.length; i++) {
byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
index 347cfb0..a5de832 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.server;
import static org.junit.Assert.assertTrue;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -153,7 +154,7 @@ public class TestBitRpc extends ExecTest {
int v = 0;
@Override
- public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data, ResponseSender sender)
+ public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, DrillBuf data, ResponseSender sender)
throws RpcException {
// System.out.println("Received.");
try {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
index 0161632..b5e8564 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.store;
+import io.netty.buffer.DrillBuf;
+
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -86,4 +88,9 @@ public class TestOutputMutator implements OutputMutator, Iterable<VectorWrapper<
addField(v);
return (T) v;
}
+
+ @Override
+ public DrillBuf getManagedBuffer() {
+ return allocator.buffer(255);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index cbae6ba..e584210 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -21,6 +21,7 @@ import static org.apache.drill.exec.store.parquet.TestFileGenerator.populateFiel
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import io.netty.buffer.DrillBuf;
import java.io.File;
import java.io.IOException;
@@ -148,16 +149,19 @@ public class ParquetRecordReaderTest extends BaseTestQuery{
@Test
+ @Ignore
public void testNonExistentColumnLargeFile() throws Exception {
testFull(QueryType.SQL, "select non_existent_column, non_existent_col_2 from dfs.`/tmp/customer.dict.parquet`", "", 1, 1, 150000, false);
}
@Test
+ @Ignore
public void testNonExistentColumnsSomePresentColumnsLargeFile() throws Exception {
testFull(QueryType.SQL, "select cust_key, address, non_existent_column, non_existent_col_2 from dfs.`/tmp/customer.dict.parquet`", "", 1, 1, 150000, false);
}
@Test
+ @Ignore
public void testTPCHPerformace_SF1() throws Exception {
testFull(QueryType.SQL, "select * from dfs.`/tmp/orders_part-m-00001.parquet`", "", 1, 1, 150000, false);
}
@@ -266,6 +270,11 @@ public class ParquetRecordReaderTest extends BaseTestQuery{
public boolean isNewSchema() {
return false;
}
+
+ @Override
+ public DrillBuf getManagedBuffer() {
+ return allocator.buffer(255);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 8b31958..f2f91f4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.vector.complex.writer;
import static org.junit.Assert.assertEquals;
+import io.netty.buffer.DrillBuf;
import java.io.ByteArrayOutputStream;
import java.util.List;
@@ -87,8 +88,8 @@ public class TestJsonReader {
ComplexWriterImpl writer = new ComplexWriterImpl("col", v);
writer.allocate();
-
- JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound));
+ DrillBuf buffer = allocator.buffer(255);
+ JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound), buffer);
int i =0;
List<Integer> batchSizes = Lists.newArrayList();
@@ -158,6 +159,6 @@ public class TestJsonReader {
assertEquals((repeatSize+1) * 2, total);
writer.clear();
-
+ buffer.release();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/sh/runexec
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/sh/runexec b/exec/java-exec/src/test/sh/runexec
new file mode 100755
index 0000000..7a45628
--- /dev/null
+++ b/exec/java-exec/src/test/sh/runexec
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROJECT_ROOT=../../../
+
+mvn dependency:build-classpath -f=$PROJECT_ROOT/pom.xml -Dmdep.outputFile=target/sh/cp.txt
+CP=`cat $PROJECT_ROOT/target/sh/cp.txt`
+CP=./:$CP:$PROJECT_ROOT/target/classes:$PROJECT_ROOT/target/test-classes
+java \
+-Dlogback.configurationFile=logback.xml \
+-XX:MaxDirectMemorySize=8192M \
+-XX:+UnlockDiagnosticVMOptions \
+-XX:FreqInlineSize=5000 \
+-XX:InlineSmallCode=5000 \
+-XX:MaxInlineLevel=20 \
+-XX:MaxInlineSize=5000 \
+-XX:MaxRecursiveInlineLevel=1 \
+-cp $CP \
+org.apache.drill.exec.RunRootExec $@
[21/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java b/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
new file mode 100644
index 0000000..fb70f89
--- /dev/null
+++ b/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
@@ -0,0 +1,674 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+
+import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.util.AssertionUtil;
+
+public final class DrillBuf extends AbstractByteBuf {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillBuf.class);
+
+ private final ByteBuf b;
+ private final long addr;
+ private final int offset;
+ private final boolean rootBuffer;
+
+ private volatile BufferAllocator allocator;
+ private volatile Accountor acct;
+ private volatile int length;
+ private final boolean emptyBuffer;
+ private OperatorContext context;
+ private FragmentContext fContext;
+
+
+ public DrillBuf(BufferAllocator allocator, Accountor a, UnsafeDirectLittleEndian b) {
+ super(b.maxCapacity());
+ this.b = b;
+ this.addr = b.memoryAddress();
+ this.acct = a;
+ this.length = b.capacity();
+ this.offset = 0;
+ this.rootBuffer = true;
+ this.allocator = allocator;
+ this.emptyBuffer = false;
+ }
+
+ private DrillBuf(ByteBuffer bb){
+ super(bb.remaining());
+ UnpooledUnsafeDirectByteBuf bytebuf = new UnpooledUnsafeDirectByteBuf(UnpooledByteBufAllocator.DEFAULT, bb, bb.remaining());
+ this.acct = FakeAllocator.FAKE_ACCOUNTOR;
+ this.addr = bytebuf.memoryAddress();
+ this.allocator = FakeAllocator.FAKE_ALLOCATOR;
+ this.b = bytebuf;
+ this.emptyBuffer = false;
+ this.length = bytebuf.capacity();
+ this.offset = 0;
+ this.rootBuffer = true;
+ this.writerIndex(bb.remaining());
+ }
+
+ private DrillBuf(BufferAllocator allocator, Accountor a){
+ super(0);
+ this.b = new EmptyByteBuf(allocator.getUnderlyingAllocator()).order(ByteOrder.LITTLE_ENDIAN);
+ this.allocator = allocator;
+ this.acct = a;
+ this.length = 0;
+ this.addr = 0;
+ this.rootBuffer = true;
+ this.offset = 0;
+ this.emptyBuffer = true;
+ }
+
+ private DrillBuf(DrillBuf buffer, int index, int length) {
+ super(length);
+ if (index < 0 || index > buffer.capacity() - length) {
+ throw new IndexOutOfBoundsException(buffer.toString() + ".slice(" + index + ", " + length + ')');
+ }
+ this.emptyBuffer = false;
+ this.length = length;
+ writerIndex(length);
+
+ this.b = buffer;
+ this.addr = buffer.memoryAddress() + index;
+ this.offset = index;
+ this.acct = null;
+ this.length = length;
+ this.rootBuffer = false;
+ this.allocator = buffer.allocator;
+ }
+
+ public void setOperatorContext(OperatorContext c){
+ this.context = c;
+ }
+ public void setFragmentContext(FragmentContext c){
+ this.fContext = c;
+ }
+
+ public BufferAllocator getAllocator(){
+ return allocator;
+ }
+
+ public DrillBuf reallocIfNeeded(int size){
+ if(this.capacity() >= size) return this;
+ if(context != null){
+ return context.replace(this, size);
+ }else if(fContext != null){
+ return fContext.replace(this, size);
+ }else{
+ throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
+ }
+
+ }
+
+ @Override
+ public int refCnt() {
+ return b.refCnt();
+ }
+
+ private long addr(int index) {
+ return addr + index;
+ }
+
+
+
+ private final void checkIndexD(int index) {
+ ensureAccessible();
+ if (index < 0 || index >= capacity()) {
+ throw new IndexOutOfBoundsException(String.format(
+ "index: %d (expected: range(0, %d))", index, capacity()));
+ }
+ }
+
+ private final void checkIndexD(int index, int fieldLength) {
+ ensureAccessible();
+ if (fieldLength < 0) {
+ throw new IllegalArgumentException("length: " + fieldLength + " (expected: >= 0)");
+ }
+ if (index < 0 || index > capacity() - fieldLength) {
+ throw new IndexOutOfBoundsException(String.format(
+ "index: %d, length: %d (expected: range(0, %d))", index, fieldLength, capacity()));
+ }
+ }
+
+ private void chk(int index, int width) {
+ if (AssertionUtil.isAssertionsEnabled()) {
+ checkIndexD(offset+index, width);
+ }
+ }
+
+ private void chk(int index) {
+ if (AssertionUtil.isAssertionsEnabled()) {
+ checkIndexD(index);
+ }
+ }
+
+ private void ensure(int width) {
+ if (AssertionUtil.isAssertionsEnabled()) {
+ ensureWritable(width);
+ }
+ }
+
+ public boolean transferAccounting(Accountor target) {
+ if (rootBuffer) {
+ boolean outcome = acct.transferTo(target, this, length);
+ acct = target;
+ return outcome;
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public synchronized boolean release() {
+ if (b.release() && rootBuffer) {
+ acct.release(this, length);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized boolean release(int decrement) {
+ if (b.release(decrement) && rootBuffer) {
+ acct.release(this, length);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int capacity() {
+ return length;
+ }
+
+
+ @Override
+ public synchronized ByteBuf capacity(int newCapacity) {
+ if (rootBuffer) {
+ if (newCapacity == length) {
+ return this;
+ } else if (newCapacity < length) {
+ b.capacity(newCapacity);
+ int diff = length - b.capacity();
+ acct.releasePartial(this, diff);
+ this.length = length - diff;
+ return this;
+ } else {
+ throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
+ }
+ } else {
+ throw new UnsupportedOperationException("Non root bufs doen't support changing allocations.");
+ }
+ }
+
+ @Override
+ public int maxCapacity() {
+ return length;
+ }
+
+ @Override
+ public ByteBufAllocator alloc() {
+ return b.alloc();
+ }
+
+ @Override
+ public ByteOrder order() {
+ return ByteOrder.LITTLE_ENDIAN;
+ }
+
+ @Override
+ public ByteBuf order(ByteOrder endianness) {
+ // if(endianness != ByteOrder.LITTLE_ENDIAN) throw new
+ // UnsupportedOperationException("Drill buffers only support little endian.");
+ return this;
+ }
+
+ @Override
+ public ByteBuf unwrap() {
+ return b;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return true;
+ }
+
+ @Override
+ public ByteBuf readBytes(int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf readSlice(int length) {
+ ByteBuf slice = slice(readerIndex(), length);
+ readerIndex(readerIndex() + length);
+ return slice;
+ }
+
+ @Override
+ public ByteBuf copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf copy(int index, int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf slice() {
+ return slice(readerIndex(), readableBytes());
+ }
+
+ @Override
+ public DrillBuf slice(int index, int length) {
+ DrillBuf buf = new DrillBuf(this, index, length);
+ buf.writerIndex = length;
+ return buf;
+ }
+
+ @Override
+ public DrillBuf duplicate() {
+ return new DrillBuf(this, 0, length);
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return 1;
+ }
+
+ @Override
+ public ByteBuffer nioBuffer() {
+ return nioBuffer(readerIndex(), readableBytes());
+ }
+
+ @Override
+ public ByteBuffer nioBuffer(int index, int length) {
+ return b.nioBuffer(offset + index, length);
+ }
+
+ @Override
+ public ByteBuffer internalNioBuffer(int index, int length) {
+ return b.internalNioBuffer(offset + index, length);
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers() {
+ return new ByteBuffer[]{nioBuffer()};
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(int index, int length) {
+ return new ByteBuffer[]{nioBuffer(index, length)};
+ }
+
+ @Override
+ public boolean hasArray() {
+ return b.hasArray();
+ }
+
+ @Override
+ public byte[] array() {
+ return b.array();
+ }
+
+ @Override
+ public int arrayOffset() {
+ return b.arrayOffset();
+ }
+
+ @Override
+ public boolean hasMemoryAddress() {
+ return true;
+ }
+
+ @Override
+ public long memoryAddress() {
+ return this.addr;
+ }
+
+ @Override
+ public String toString(Charset charset) {
+ return toString(readerIndex, readableBytes(), charset);
+ }
+
+ @Override
+ public String toString(int index, int length, Charset charset) {
+ if (length == 0) {
+ return "";
+ }
+
+ ByteBuffer nioBuffer;
+ if (nioBufferCount() == 1) {
+ nioBuffer = nioBuffer(index, length);
+ } else {
+ nioBuffer = ByteBuffer.allocate(length);
+ getBytes(index, nioBuffer);
+ nioBuffer.flip();
+ }
+
+ return ByteBufUtil.decodeString(nioBuffer, charset);
+ }
+
+ @Override
+ public int hashCode() {
+ return System.identityHashCode(this);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // identity equals only.
+ return this == obj;
+ }
+
+ @Override
+ public ByteBuf retain(int increment) {
+ b.retain(increment);
+ return this;
+ }
+
+ @Override
+ public ByteBuf retain() {
+ b.retain();
+ return this;
+ }
+
+ @Override
+ public long getLong(int index) {
+ chk(index, 8);
+ long v = PlatformDependent.getLong(addr(index));
+ return v;
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return Float.intBitsToFloat(getInt(index));
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return Double.longBitsToDouble(getLong(index));
+ }
+
+ @Override
+ public char getChar(int index) {
+ return (char) getShort(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return getInt(index) & 0xFFFFFFFFL;
+ }
+
+ @Override
+ public int getInt(int index) {
+ chk(index, 4);
+ int v = PlatformDependent.getInt(addr(index));
+ return v;
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return getShort(index) & 0xFFFF;
+ }
+
+ @Override
+ public short getShort(int index) {
+ chk(index, 2);
+ short v = PlatformDependent.getShort(addr(index));
+ return v;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ chk(index, 2);
+ PlatformDependent.putShort(addr(index), (short) value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ chk(index, 4);
+ PlatformDependent.putInt(addr(index), value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ chk(index, 8);
+ PlatformDependent.putLong(addr(index), value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ chk(index, 2);
+ PlatformDependent.putShort(addr(index), (short) value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ chk(index, 4);
+ PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ chk(index, 8);
+ PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ ensure(2);
+ PlatformDependent.putShort(addr(writerIndex), (short) value);
+ writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ ensure(4);
+ PlatformDependent.putInt(addr(writerIndex), value);
+ writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ ensure(8);
+ PlatformDependent.putLong(addr(writerIndex), value);
+ writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ ensure(2);
+ PlatformDependent.putShort(addr(writerIndex), (short) value);
+ writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ ensure(4);
+ PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
+ writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ ensure(8);
+ PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
+ writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+ b.getBytes(index + offset, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuffer dst) {
+ b.getBytes(index + offset, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ PlatformDependent.putByte(addr(index), (byte) value);
+ return this;
+ }
+
+ @Override
+ protected byte _getByte(int index) {
+ return getByte(index);
+ }
+
+ @Override
+ protected short _getShort(int index) {
+ return getShort(index);
+ }
+
+ @Override
+ protected int _getInt(int index) {
+ return getInt(index);
+ }
+
+ @Override
+ protected long _getLong(int index) {
+ return getLong(index);
+ }
+
+ @Override
+ protected void _setByte(int index, int value) {
+ setByte(index, value);
+ }
+
+ @Override
+ protected void _setShort(int index, int value) {
+ setShort(index, value);
+ }
+
+ @Override
+ protected void _setMedium(int index, int value) {
+ setMedium(index, value);
+ }
+
+ @Override
+ protected void _setInt(int index, int value) {
+ setInt(index, value);
+ }
+
+ @Override
+ protected void _setLong(int index, long value) {
+ setLong(index, value);
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+ b.getBytes(index + offset, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+ b.getBytes(index + offset, out, length);
+ return this;
+ }
+
+ @Override
+ protected int _getUnsignedMedium(int index) {
+ long addr = addr(index);
+ return (PlatformDependent.getByte(addr) & 0xff) << 16 |
+ (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
+ PlatformDependent.getByte(addr + 2) & 0xff;
+ }
+
+ @Override
+ public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
+ return b.getBytes(index + offset, out, length);
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+ b.setBytes(index + offset, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+ b.setBytes(index + offset, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuffer src) {
+ b.setBytes(index + offset, src);
+ return this;
+ }
+
+ @Override
+ public int setBytes(int index, InputStream in, int length) throws IOException {
+ return b.setBytes(index + offset, in, length);
+ }
+
+ @Override
+ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
+ return b.setBytes(index + offset, in, length);
+ }
+
+ @Override
+ public byte getByte(int index) {
+ return PlatformDependent.getByte(addr(index));
+ }
+
+ public static DrillBuf getEmpty(BufferAllocator allocator, Accountor a){
+ return new DrillBuf(allocator, a);
+ }
+
+ public static DrillBuf wrapByteBuffer(ByteBuffer b){
+ if(!b.isDirect()){
+ throw new IllegalStateException("DrillBufs can only refer to direct memory.");
+ }else{
+ return new DrillBuf(b);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
new file mode 100644
index 0000000..bc69577
--- /dev/null
+++ b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+
+class FakeAllocator implements BufferAllocator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FakeAllocator.class);
+
+
+ public static final Accountor FAKE_ACCOUNTOR = new FakeAccountor();
+ public static final BufferAllocator FAKE_ALLOCATOR = new FakeAllocator();
+
+ @Override
+ public DrillBuf buffer(int size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DrillBuf buffer(int minSize, int maxSize) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBufAllocator getUnderlyingAllocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BufferAllocator getChildAllocator(FragmentHandle handle, long initialReservation, long maximumReservation)
+ throws OutOfMemoryException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DrillBuf getEmpty() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public PreAllocator getNewPreAllocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long getAllocatedMemory() {
+ return 0;
+ }
+
+ static class FakeAccountor extends Accountor {
+
+ public FakeAccountor() {
+ super(false, null, null, 0, 0);
+ }
+
+ @Override
+ public boolean transferTo(Accountor target, DrillBuf buf, long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getAvailable() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getCapacity() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getAllocation() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean reserve(long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean forceAdditionalReservation(long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reserved(long expected, DrillBuf buf) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void releasePartial(DrillBuf buf, long size) {
+
+ }
+
+ @Override
+ public void release(DrillBuf buf, long size) {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index a8482ae..c0de544 100644
--- a/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -58,7 +58,7 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
@Override
- public ByteBuf directBuffer(int initialCapacity, int maxCapacity) {
+ public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
if (initialCapacity == 0 && maxCapacity == 0) {
newDirectBuffer(initialCapacity, maxCapacity);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index ad85a63..86d3b59 100644
--- a/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -22,7 +22,9 @@ import io.netty.util.internal.PlatformDependent;
import java.nio.ByteOrder;
-final class UnsafeDirectLittleEndian extends WrappedByteBuf {
+import org.apache.drill.exec.memory.BufferAllocator;
+
+public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
private final PooledUnsafeDirectByteBuf wrapped;
private final long memoryAddress;
@@ -40,7 +42,7 @@ final class UnsafeDirectLittleEndian extends WrappedByteBuf {
@Override
public long getLong(int index) {
- wrapped.checkIndex(index, 8);
+// wrapped.checkIndex(index, 8);
long v = PlatformDependent.getLong(addr(index));
return v;
}
@@ -67,7 +69,7 @@ final class UnsafeDirectLittleEndian extends WrappedByteBuf {
@Override
public int getInt(int index) {
- wrapped.checkIndex(index, 4);
+// wrapped.checkIndex(index, 4);
int v = PlatformDependent.getInt(addr(index));
return v;
}
@@ -79,7 +81,7 @@ final class UnsafeDirectLittleEndian extends WrappedByteBuf {
@Override
public short getShort(int index) {
- wrapped.checkIndex(index, 2);
+// wrapped.checkIndex(index, 2);
short v = PlatformDependent.getShort(addr(index));
return v;
}
@@ -176,4 +178,18 @@ final class UnsafeDirectLittleEndian extends WrappedByteBuf {
private void _setLong(int index, long value) {
PlatformDependent.putLong(addr(index), value);
}
+
+ @Override
+ public byte getByte(int index) {
+ return PlatformDependent.getByte(addr(index));
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ PlatformDependent.putByte(addr(index), (byte) value);
+ return this;
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/cache/VectorAccessibleSerializable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/VectorAccessibleSerializable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/VectorAccessibleSerializable.java
index cff56d6..e007bcc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/VectorAccessibleSerializable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/VectorAccessibleSerializable.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.cache;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.io.InputStream;
@@ -112,7 +113,7 @@ public class VectorAccessibleSerializable extends AbstractStreamSerializable {
for (SerializedField metaData : fieldList) {
int dataLength = metaData.getBufferLength();
MaterializedField field = MaterializedField.create(metaData);
- ByteBuf buf = allocator.buffer(dataLength);
+ DrillBuf buf = allocator.buffer(dataLength);
buf.writeBytes(input, dataLength);
ValueVector vector = TypeHelper.getNewVector(field, allocator);
vector.load(metaData, buf);
@@ -146,7 +147,7 @@ public class VectorAccessibleSerializable extends AbstractStreamSerializable {
UserBitShared.RecordBatchDef batchDef = batch.getDef();
/* ByteBuf associated with the selection vector */
- ByteBuf svBuf = null;
+ DrillBuf svBuf = null;
Integer svCount = null;
if (svMode == BatchSchema.SelectionVectorMode.TWO_BYTE)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ByteCodeLoader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ByteCodeLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ByteCodeLoader.java
index 36db0ed..6488699 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ByteCodeLoader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ByteCodeLoader.java
@@ -31,8 +31,8 @@ import com.google.common.io.Resources;
class ByteCodeLoader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteCodeLoader.class);
-
-
+
+
private final LoadingCache<String, byte[]> byteCode = CacheBuilder.newBuilder().maximumSize(10000)
.expireAfterWrite(10, TimeUnit.MINUTES).build(new ClassBytesCacheLoader());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
index 44ea5b5..bb24b57 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
@@ -21,10 +21,10 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.compile.MergeAdapter.MergedClassResult;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.codehaus.commons.compiler.CompileException;
@@ -39,30 +39,12 @@ import com.google.common.collect.Sets;
public class ClassTransformer {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassTransformer.class);
- private ByteCodeLoader byteCodeLoader = new ByteCodeLoader();
- private AtomicLong index = new AtomicLong(0);
-
-
-// public <T, I> T getImplementationClassByBody( QueryClassLoader classLoader, TemplateClassDefinition<T> templateDefinition, //
-// String internalClassBody //
-// ) throws ClassTransformationException {
-// final String materializedClassName = "org.apache.drill.generated."
-// + "Gen" + templateDefinition.getExternalInterface().getSimpleName() //
-// + index.getAndIncrement();
-// // Get Implementation Class
-// try {
-// String classBody = ClassBodyBuilder.newBuilder() //
-// .setClassName(materializedClassName) //
-// .setBody(internalClassBody) //
-// .build();
-// return getImplementationClass(classLoader, templateDefinition, classBody, materializedClassName);
-// } catch (IOException | CompileException e) {
-// throw new ClassTransformationException("Failure generating class body for runtime generated class.", e);
-// }
-//
-// }
-
+ private final ByteCodeLoader byteCodeLoader = new ByteCodeLoader();
+ private final DistributedCache cache;
+ public ClassTransformer(DistributedCache cache) {
+ this.cache = cache;
+ }
public static class ClassSet{
public final ClassSet parent;
@@ -175,65 +157,8 @@ public class ClassTransformer {
return false;
return true;
}
-
-
-//
-// public ClassNames getFixed(ClassNames precompiled, ClassNames generated){
-// if(!dot.startsWith(precompiled.dot)) throw new IllegalStateException(String.format("Expected a class that starts with %s. However the class %s does not start with this string.", precompiled.dot, dot));
-// return new ClassNames(dot.replace(precompiled.dot, generated.dot));
-// }
}
-//
-// private void mergeAndInjectClass(QueryClassLoader classLoader, byte[] implementationClass, ClassNames precompiled, ClassNames generated){
-// // Get Template Class
-// final byte[] templateClass = byteCodeLoader.getClassByteCodeFromPath(precompiled.clazz);
-//
-// // get the runtime generated class's ClassNode from bytecode.
-// ClassNode impl = getClassNodeFromByteCode(implementationClass);
-//
-// // Setup adapters for merging, remapping class names and class writing. This is done in reverse order of how they
-// // will be evaluated.
-//
-// Stopwatch t3;
-// {
-//
-// //
-// ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
-//
-// ClassVisitor remappingAdapter = new RemappingClassAdapter(cw, remapper);
-// MergeAdapter mergingAdapter = new MergeAdapter(oldTemplateSlashName, materializedSlashName, remappingAdapter,
-// impl);
-// ClassReader tReader = new ClassReader(templateClass);
-// tReader.accept(mergingAdapter, ClassReader.EXPAND_FRAMES);
-// byte[] outputClass = cw.toByteArray();
-//// Files.write(outputClass, new File(String.format("/tmp/%d-output.class", fileNum)));
-// outputClass = cw.toByteArray();
-//
-// // Load the class
-// classLoader.injectByteCode(materializedClassName, outputClass);
-// }
-// t3.stop();
-// Stopwatch t4 = new Stopwatch().start();
-// int i = 0;
-// for (String s : remapper.getInnerClasses()) {
-// logger.debug("Setting up sub class {}", s);
-// // for each sub class, remap them into the new class.
-// String subclassPath = FileUtils.separator + s + ".class";
-// final byte[] bytecode = getClassByteCodeFromPath(subclassPath);
-// RemapClasses localRemapper = new RemapClasses(oldTemplateSlashName, materializedSlashName);
-// Preconditions.checkArgument(localRemapper.getInnerClasses().isEmpty(), "Class transformations are only supported for classes that have a single level of inner classes.");
-// ClassWriter subcw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
-// ClassVisitor remap = new RemappingClassAdapter(subcw, localRemapper);
-// ClassReader reader = new ClassReader(bytecode);
-// reader.accept(remap, ClassReader.EXPAND_FRAMES);
-// byte[] newByteCode = subcw.toByteArray();
-// classLoader.injectByteCode(s.replace(oldTemplateSlashName, materializedSlashName).replace(FileUtils.separatorChar, '.'), newByteCode);
-//// Files.write(subcw.toByteArray(), new File(String.format("/tmp/%d-sub-%d.class", fileNum, i)));
-// i++;
-// }
-// }
-//
private static ClassNode getClassNodeFromByteCode(byte[] bytes) {
ClassReader iReader = new ClassReader(bytes);
ClassNode impl = new ClassNode();
@@ -241,10 +166,9 @@ public class ClassTransformer {
return impl;
}
- @SuppressWarnings("unchecked")
- public <T, I> T getImplementationClass( //
+ public Class<?> getImplementationClass( //
QueryClassLoader classLoader, //
- TemplateClassDefinition<T> templateDefinition, //
+ TemplateClassDefinition<?> templateDefinition, //
String entireClass, //
String materializedClassName) throws ClassTransformationException {
@@ -284,17 +208,16 @@ public class ClassTransformer {
Class<?> c = classLoader.findClass(set.generated.dot);
if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
- T instance = (T) c.newInstance();
- logger.debug("Done compiling (bytecode size={}, time:{} millis).",
- DrillStringUtils.readable(totalBytecodeSize), (System.nanoTime() - t1) / 1000000);
- return instance;
+ logger.debug("Done compiling (bytecode size={}, time:{} millis).", DrillStringUtils.readable(totalBytecodeSize), (System.nanoTime() - t1) / 1000000);
+ return c;
} else {
throw new ClassTransformationException("The requested class did not implement the expected interface.");
}
- } catch (CompileException | IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+ } catch (CompileException | IOException | ClassNotFoundException e) {
throw new ClassTransformationException(String.format("Failure generating transformation classes for value: \n %s", entireClass), e);
}
}
-}
\ No newline at end of file
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
new file mode 100644
index 0000000..2edc902
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.cache.DistributedCache;
+import org.apache.drill.exec.cache.local.LocalCache;
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+public class CodeCompiler {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeCompiler.class);
+
+ private final ClassTransformer transformer;
+ private final DistributedCache distributedCache;
+ private final LoadingCache<CodeGenerator<?>, GeneratedClassEntry> cache;
+ private final DrillConfig config;
+ private final OptionManager systemOptionManager;
+
+ public CodeCompiler(DrillConfig config, DistributedCache distributedCache, OptionManager systemOptionManager){
+ this.transformer = new ClassTransformer(distributedCache);
+ this.distributedCache = distributedCache;
+ this.cache = CacheBuilder.newBuilder().build(new Loader());
+ this.systemOptionManager = systemOptionManager;
+ this.config = config;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T getImplementationClass(CodeGenerator<?> cg) throws ClassTransformationException, IOException {
+ cg.generate();
+ try {
+ GeneratedClassEntry ce = cache.get(cg);
+ return (T) ce.clazz.newInstance();
+ } catch (ExecutionException | InstantiationException | IllegalAccessException e) {
+ throw new ClassTransformationException(e);
+ }
+ }
+
+ private class Loader extends CacheLoader<CodeGenerator<?>, GeneratedClassEntry>{
+ @Override
+ public GeneratedClassEntry load(CodeGenerator<?> cg) throws Exception {
+ QueryClassLoader loader = new QueryClassLoader(config, systemOptionManager);
+ Class<?> c = transformer.getImplementationClass(loader, cg.getDefinition(), cg.getGeneratedCode(), cg.getMaterializedClassName());
+ return new GeneratedClassEntry(loader, c);
+ }
+ }
+
+ private class GeneratedClassEntry {
+
+ private final QueryClassLoader classLoader;
+ private final Class<?> clazz;
+
+ public GeneratedClassEntry(QueryClassLoader classLoader, Class<?> c) {
+ super();
+ this.classLoader = classLoader;
+ this.clazz = c;
+ }
+
+ }
+
+ public static CodeCompiler getTestCompiler(DrillConfig c) throws IOException{
+ return new CodeCompiler(c, new LocalCache(), new SystemOptionManager(c, new LocalPStoreProvider(c)).init());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillInitMethodVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillInitMethodVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillInitMethodVisitor.java
new file mode 100644
index 0000000..859a14c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillInitMethodVisitor.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile;
+
+import org.apache.drill.exec.compile.sig.SignatureHolder;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+public class DrillInitMethodVisitor extends MethodVisitor {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillInitMethodVisitor.class);
+
+ final String className;
+
+ public DrillInitMethodVisitor(String className, MethodVisitor mv){
+ super(Opcodes.ASM4, mv);
+ this.className = className;
+ }
+
+ @Override
+ public void visitInsn(int opcode) {
+ if(opcode == Opcodes.RETURN){
+ super.visitVarInsn(Opcodes.ALOAD, 0); // load this.
+ super.visitMethodInsn(Opcodes.INVOKEVIRTUAL, className, SignatureHolder.DRILL_INIT_METHOD, "()V"); // execute drill init.
+ }
+ super.visitInsn(opcode);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
index 30cde91..bd68b43 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
@@ -17,15 +17,17 @@
*/
package org.apache.drill.exec.compile;
-import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.lang.reflect.Modifier;
import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
-import org.apache.drill.exec.compile.ClassTransformer.ClassNames;
import org.apache.drill.exec.compile.ClassTransformer.ClassSet;
+import org.apache.drill.exec.compile.bytecode.ValueHolderReplacementVisitor;
+import org.apache.drill.exec.compile.sig.SignatureHolder;
import org.objectweb.asm.AnnotationVisitor;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassVisitor;
@@ -40,10 +42,10 @@ import org.objectweb.asm.commons.SimpleRemapper;
import org.objectweb.asm.tree.ClassNode;
import org.objectweb.asm.tree.FieldNode;
import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.util.CheckClassAdapter;
+import org.objectweb.asm.util.TraceClassVisitor;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
-import com.google.common.io.Files;
/**
* Serves two purposes. Renames all inner classes references to the outer class to the new name. Also adds all the
@@ -55,8 +57,9 @@ class MergeAdapter extends ClassVisitor {
private ClassNode classToMerge;
private ClassSet set;
-
private Set<String> mergingNames = Sets.newHashSet();
+ private boolean hasInit;
+ private String name;
private MergeAdapter(ClassSet set, ClassVisitor cv, ClassNode cn) {
super(Opcodes.ASM4, cv);
@@ -65,6 +68,7 @@ class MergeAdapter extends ClassVisitor {
for(Object o : classToMerge.methods){
String name = ((MethodNode)o).name;
if(name.equals("<init>")) continue;
+ if(name.equals(SignatureHolder.DRILL_INIT_METHOD)) hasInit = true;
mergingNames.add( name);
}
}
@@ -73,6 +77,7 @@ class MergeAdapter extends ClassVisitor {
public void visitInnerClass(String name, String outerName, String innerName, int access) {
// logger.debug(String.format("[Inner Class] Name: %s, outerName: %s, innerName: %s, templateName: %s, newName: %s.",
// name, outerName, innerName, templateName, newName));
+
if (name.startsWith(set.precompiled.slash)) {
// outerName = outerName.replace(precompiled.slash, generated.slash);
name = name.replace(set.precompiled.slash, set.generated.slash);
@@ -92,6 +97,7 @@ class MergeAdapter extends ClassVisitor {
// visit the class
public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
// use the access and names of the impl class.
+ this.name = name;
if(name.contains("$")){
super.visit(version, access, name, signature, superName, interfaces);
}else{
@@ -102,25 +108,32 @@ class MergeAdapter extends ClassVisitor {
}
@Override
- public MethodVisitor visitMethod(int access, String arg1, String arg2, String arg3, String[] arg4) {
+ public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+
+
// finalize all methods.
// skip all abstract methods as they should have implementations.
- if ((access & Modifier.ABSTRACT) != 0 || mergingNames.contains(arg1)) {
+ if ((access & Modifier.ABSTRACT) != 0 || mergingNames.contains(name)) {
// logger.debug("Skipping copy of '{}()' since it is abstract or listed elsewhere.", arg1);
return null;
}
- if(arg3 != null){
- arg3 = arg3.replace(set.precompiled.slash, set.generated.slash);
+ if(signature != null){
+ signature = signature.replace(set.precompiled.slash, set.generated.slash);
}
// if( (access & Modifier.PUBLIC) == 0){
// access = access ^ Modifier.PUBLIC ^ Modifier.PROTECTED | Modifier.PRIVATE;
// }
- if (!arg1.equals("<init>")) {
+ MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions);
+ if (!name.equals("<init>")) {
access = access | Modifier.FINAL;
+ }else{
+ if(hasInit){
+ return new DrillInitMethodVisitor(this.name, mv);
+ }
}
- return super.visitMethod(access, arg1, arg2, arg3, arg4);
+ return mv;
}
@SuppressWarnings("unchecked")
@@ -135,19 +148,20 @@ class MergeAdapter extends ClassVisitor {
for (Iterator<?> it = classToMerge.methods.iterator(); it.hasNext();) {
MethodNode mn = (MethodNode) it.next();
- // skip the init.
- if (mn.name.equals("<init>"))
- continue;
+ if (mn.name.equals("<init>")) continue;
String[] exceptions = new String[mn.exceptions.size()];
mn.exceptions.toArray(exceptions);
- MethodVisitor mv = cv.visitMethod(mn.access | Modifier.FINAL, mn.name, mn.desc, mn.signature, exceptions);
+ MethodVisitor mv = cv.visitMethod(mn.access | Modifier.FINAL, mn.name, mn.desc, mn.signature, exceptions);
+
mn.instructions.resetLabels();
+
// mn.accept(new RemappingMethodAdapter(mn.access, mn.desc, mv, new
// SimpleRemapper("org.apache.drill.exec.compile.ExampleTemplate", "Bunky")));
ClassSet top = set;
while(top.parent != null) top = top.parent;
mn.accept(new RemappingMethodAdapter(mn.access, mn.desc, mv, new SimpleRemapper(top.precompiled.slash, top.generated.slash)));
+
}
super.visitEnd();
}
@@ -175,21 +189,31 @@ class MergeAdapter extends ClassVisitor {
// Setup adapters for merging, remapping class names and class writing. This is done in reverse order of how they
// will be evaluated.
- ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
+ ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
RemapClasses re = new RemapClasses(set);
- ClassVisitor remappingAdapter = new RemappingClassAdapter(cw, re);
- ClassVisitor visitor = remappingAdapter;
- if(generatedClass != null){
- visitor = new MergeAdapter(set, remappingAdapter, generatedClass);
- }
- ClassReader tReader = new ClassReader(precompiledClass);
- tReader.accept(visitor, ClassReader.EXPAND_FRAMES);
- byte[] outputClass = cw.toByteArray();
+ try{
+// if(generatedClass != null){
+// ClassNode generatedMerged = new ClassNode();
+// generatedClass.accept(new ValueHolderReplacementVisitor(generatedMerged));
+// generatedClass = generatedMerged;
+// }
+ ClassVisitor remappingAdapter = new RemappingClassAdapter(writer, re);
+ ClassVisitor visitor = remappingAdapter;
+ if(generatedClass != null){
+ visitor = new MergeAdapter(set, remappingAdapter, generatedClass);
+ }
+ ClassReader tReader = new ClassReader(precompiledClass);
+ tReader.accept(visitor, ClassReader.SKIP_FRAMES);
+ byte[] outputClass = writer.toByteArray();
- // enable when you want all the generated merged class files to also be written to disk.
- //Files.write(outputClass, new File(String.format("/tmp/drill-generated-classes/%s-output.class", set.generated.dot)));
+ // enable when you want all the generated merged class files to also be written to disk.
+// Files.write(outputClass, new File(String.format("/src/scratch/drill-generated-classes/%s-output.class", set.generated.dot)));
- return new MergedClassResult(outputClass, re.getInnerClasses());
+ return new MergedClassResult(outputClass, re.getInnerClasses());
+ }catch(Error | RuntimeException e){
+ logger.error("Failure while merging classes.", e);
+ throw e;
+ }
}
@@ -227,4 +251,36 @@ class MergeAdapter extends ClassVisitor {
}
}
+
+ private static final void check(ClassNode node) {
+ Exception e = null;
+ String error = "";
+
+ try{
+ ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
+ ClassVisitor cv = new CheckClassAdapter(cw, true);
+ node.accept(cv);
+
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ CheckClassAdapter.verify(new ClassReader(cw.toByteArray()), false, pw);
+
+ error = sw.toString();
+ }catch(Exception ex){
+ e = ex;
+ }
+
+ if(!error.isEmpty() || e != null){
+ StringWriter sw2 = new StringWriter();
+ PrintWriter pw2 = new PrintWriter(sw2);
+ TraceClassVisitor v = new TraceClassVisitor(pw2);
+ node.accept(v);
+ if(e != null){
+ throw new RuntimeException("Failure validating class. ByteCode: \n" + sw2.toString() + "\n\n====ERRROR====\n" + error, e);
+ }else{
+ throw new RuntimeException("Failure validating class. ByteCode: \n" + sw2.toString() + "\n\n====ERRROR====\n" + error);
+ }
+
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
index 688e680..2b6ddf0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
@@ -22,14 +22,14 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.exec.compile.sig.SignatureHolder;
public class TemplateClassDefinition<T>{
-
+
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TemplateClassDefinition.class);
-
+
private final Class<T> iface;
private final Class<?> template;
private final SignatureHolder signature;
private static final AtomicLong classNumber = new AtomicLong(0);
-
+
public <X extends T> TemplateClassDefinition(Class<T> iface, Class<X> template) {
super();
this.iface = iface;
@@ -47,7 +47,7 @@ public class TemplateClassDefinition<T>{
public long getNextClassNumber(){
return classNumber.getAndIncrement();
}
-
+
public Class<T> getExternalInterface() {
return iface;
}
@@ -66,6 +66,11 @@ public class TemplateClassDefinition<T>{
return "TemplateClassDefinition [template=" + template + ", signature=" + signature + "]";
}
-
-
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/DirectSorter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/DirectSorter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/DirectSorter.java
new file mode 100644
index 0000000..4e54bc7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/DirectSorter.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.commons.LocalVariablesSorter;
+
+public class DirectSorter extends LocalVariablesSorter{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DirectSorter.class);
+
+ public DirectSorter(int access, String desc, MethodVisitor mv) {
+ super(access, desc, mv);
+ }
+
+ public void directVarInsn(int opcode, int var) {
+ mv.visitVarInsn(opcode, var);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/InstructionModifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/InstructionModifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/InstructionModifier.java
new file mode 100644
index 0000000..e736aab
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/InstructionModifier.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import org.apache.drill.exec.compile.bytecode.ValueHolderIden.ValueHolderSub;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+import com.carrotsearch.hppc.IntIntOpenHashMap;
+import com.carrotsearch.hppc.IntObjectOpenHashMap;
+
+public class InstructionModifier extends MethodVisitor {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InstructionModifier.class);
+
+ private final IntObjectOpenHashMap<ValueHolderIden.ValueHolderSub> oldToNew = new IntObjectOpenHashMap<>();
+ private final IntIntOpenHashMap oldLocalToFirst = new IntIntOpenHashMap();
+
+ private DirectSorter adder;
+ int lastLineNumber = 0;
+ private TrackingInstructionList list;
+
+ public InstructionModifier(int access, String name, String desc, String signature, String[] exceptions,
+ TrackingInstructionList list, MethodVisitor inner) {
+ super(Opcodes.ASM4, new DirectSorter(access, desc, inner));
+ this.list = list;
+ this.adder = (DirectSorter) mv;
+ }
+
+ public void setList(TrackingInstructionList list) {
+ this.list = list;
+ }
+
+ private ReplacingBasicValue local(int var) {
+ Object o = list.currentFrame.getLocal(var);
+ if (o instanceof ReplacingBasicValue) {
+ return (ReplacingBasicValue) o;
+ }
+ return null;
+ }
+
+ private ReplacingBasicValue popCurrent() {
+ return popCurrent(false);
+ }
+
+ private ReplacingBasicValue popCurrent(boolean includeReturnVals) {
+ // for vararg, we could try to pop an empty stack. TODO: handle this better.
+ if (list.currentFrame.getStackSize() == 0)
+ return null;
+
+ Object o = list.currentFrame.pop();
+ if (o instanceof ReplacingBasicValue) {
+ ReplacingBasicValue v = (ReplacingBasicValue) o;
+ if (!v.isFunctionReturn || includeReturnVals) {
+ return v;
+ }
+ }
+ return null;
+ }
+
+ private ReplacingBasicValue getReturn() {
+ Object o = list.nextFrame.getStack(list.nextFrame.getStackSize() - 1);
+ if (o instanceof ReplacingBasicValue)
+ return (ReplacingBasicValue) o;
+ return null;
+ }
+
+ @Override
+ public void visitInsn(int opcode) {
+ switch (opcode) {
+ case Opcodes.DUP:
+ if (popCurrent() != null)
+ return;
+ }
+ super.visitInsn(opcode);
+ }
+
+ @Override
+ public void visitTypeInsn(int opcode, String type) {
+ ReplacingBasicValue r = getReturn();
+ if (r != null) {
+ ValueHolderSub sub = r.getIden().getHolderSub(adder);
+ oldToNew.put(r.getIndex(), sub);
+ } else {
+ super.visitTypeInsn(opcode, type);
+ }
+ }
+
+ @Override
+ public void visitLineNumber(int line, Label start) {
+ lastLineNumber = line;
+ super.visitLineNumber(line, start);
+ }
+
+ @Override
+ public void visitVarInsn(int opcode, int var) {
+ ReplacingBasicValue v;
+ if(opcode == Opcodes.ASTORE && (v = popCurrent(true)) != null){
+ if(!v.isFunctionReturn){
+ ValueHolderSub from = oldToNew.get(v.getIndex());
+
+ ReplacingBasicValue current = local(var);
+ // if local var is set, then transfer to it to the existing holders in the local position.
+ if(current != null){
+ if(oldToNew.get(current.getIndex()).iden() == from.iden()){
+ int targetFirst = oldToNew.get(current.index).first();
+ from.transfer(this, targetFirst);
+ return;
+ }
+ }
+
+ // if local var is not set, then check map to see if existing holders are mapped to local var.
+ if(oldLocalToFirst.containsKey(var)){
+ ValueHolderSub sub = oldToNew.get(oldLocalToFirst.lget());
+ if(sub.iden() == from.iden()){
+ // if they are, then transfer to that.
+ from.transfer(this, oldToNew.get(oldLocalToFirst.lget()).first());
+ return;
+ }
+ }
+
+
+ // map from variables to global space for future use.
+ oldLocalToFirst.put(var, v.getIndex());
+
+ }else{
+ // this is storage of a function return, we need to map the fields to the holder spots.
+ int first;
+ if(oldLocalToFirst.containsKey(var)){
+ first = oldToNew.get(oldLocalToFirst.lget()).first();
+ v.iden.transferToLocal(adder, first);
+ }else{
+ first = v.iden.createLocalAndTrasfer(adder);
+ }
+ ValueHolderSub from = v.iden.getHolderSubWithDefinedLocals(first);
+ oldToNew.put(v.getIndex(), from);
+ v.disableFunctionReturn();
+ }
+
+ }else if(opcode == Opcodes.ALOAD && (v = getReturn()) != null){
+
+ // noop.
+ }else{
+ super.visitVarInsn(opcode, var);
+ }
+
+
+ }
+
+ void directVarInsn(int opcode, int var) {
+ adder.directVarInsn(opcode, var);
+ }
+
+ @Override
+ public void visitFieldInsn(int opcode, String owner, String name, String desc) {
+ ReplacingBasicValue v;
+
+ switch (opcode) {
+ case Opcodes.PUTFIELD:
+ // pop twice for put.
+ v = popCurrent(true);
+ if (v != null) {
+ if(v.isFunctionReturn){
+ super.visitFieldInsn(opcode, owner, name, desc);
+ return;
+ }else{
+ // we are trying to store a replaced variable in an external context, we need to generate an instance and
+ // transfer it out.
+ ValueHolderSub sub = oldToNew.get(v.getIndex());
+ sub.transferToExternal(adder, owner, name, desc);
+ return;
+ }
+ }
+
+ case Opcodes.GETFIELD:
+ // pop again.
+ v = popCurrent();
+ if (v != null) {
+ // super.visitFieldInsn(opcode, owner, name, desc);
+ ValueHolderSub sub = oldToNew.get(v.getIndex());
+ sub.addInsn(name, this, opcode);
+ return;
+ }
+
+
+ }
+
+ super.visitFieldInsn(opcode, owner, name, desc);
+ }
+
+ @Override
+ public void visitLocalVariable(String name, String desc, String signature, Label start, Label end, int index) {
+ // if (!ReplacingInterpreter.HOLDER_DESCRIPTORS.contains(desc)) {
+ // super.visitLocalVariable(name, desc, signature, start, end, index);
+ // }
+ }
+
+ @Override
+ public void visitMethodInsn(int opcode, String owner, String name, String desc) {
+ final int len = Type.getArgumentTypes(desc).length;
+ final boolean isStatic = opcode == Opcodes.INVOKESTATIC;
+
+ ReplacingBasicValue obj = popCurrent();
+
+ if (obj != null && !isStatic) {
+ if ("<init>".equals(name)) {
+ oldToNew.get(obj.getIndex()).init(adder);
+ } else {
+ throw new IllegalStateException("you can't call a method on a value holder.");
+ }
+ return;
+ }
+
+ obj = getReturn();
+
+ if (obj != null) {
+ // the return of this method is an actual instance of the object we're escaping. Update so that it get's mapped
+ // correctly.
+ super.visitMethodInsn(opcode, owner, name, desc);
+ obj.markFunctionReturn();
+ return;
+ }
+
+ int i = isStatic ? 1 : 0;
+ for (; i < len; i++) {
+ checkArg(name, popCurrent());
+ }
+
+ super.visitMethodInsn(opcode, owner, name, desc);
+ }
+
+ private void checkArg(String name, ReplacingBasicValue obj) {
+ if (obj == null)
+ return;
+ throw new IllegalStateException(
+ String
+ .format(
+ "Holder types are not allowed to be passed between methods. Ran across problem attempting to invoke method '%s' on line number %d",
+ name, lastLineNumber));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingBasicValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingBasicValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingBasicValue.java
new file mode 100644
index 0000000..8e05602
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingBasicValue.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import org.objectweb.asm.Type;
+import org.objectweb.asm.tree.analysis.BasicValue;
+
+public class ReplacingBasicValue extends BasicValue{
+
+ ValueHolderIden iden;
+ int index;
+ Type type;
+ boolean isFunctionReturn = false;
+
+ public ReplacingBasicValue(Type type, ValueHolderIden iden, int index) {
+ super(type);
+ this.index = index;
+ this.iden = iden;
+ this.type = type;
+ }
+
+ public void markFunctionReturn(){
+ this.isFunctionReturn = true;
+ }
+
+ public void disableFunctionReturn(){
+ this.isFunctionReturn = false;
+ }
+
+ public ValueHolderIden getIden() {
+ return iden;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public Type getType(){
+ return type;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java
new file mode 100644
index 0000000..2dce4db
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.tree.AbstractInsnNode;
+import org.objectweb.asm.tree.TypeInsnNode;
+import org.objectweb.asm.tree.analysis.AnalyzerException;
+import org.objectweb.asm.tree.analysis.BasicInterpreter;
+import org.objectweb.asm.tree.analysis.BasicValue;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+public class ReplacingInterpreter extends BasicInterpreter {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReplacingInterpreter.class);
+
+ private int index = 0;
+
+ @Override
+ public BasicValue newValue(Type t) {
+ if(t != null){
+ ValueHolderIden iden = HOLDERS.get(t.getDescriptor());
+ if(iden != null){
+ ReplacingBasicValue v = new ReplacingBasicValue(t, iden, index++);
+ v.markFunctionReturn();
+ return v;
+ }
+ }
+ return super.newValue(t);
+
+ }
+
+ @Override
+ public BasicValue newOperation(AbstractInsnNode insn) throws AnalyzerException {
+ if(insn.getOpcode() == Opcodes.NEW){
+ TypeInsnNode t = (TypeInsnNode) insn;
+ ValueHolderIden iden = HOLDERS.get(t.desc);
+
+ if(iden != null){
+ return new ReplacingBasicValue(Type.getObjectType(t.desc), iden, index++);
+ }
+ }
+
+ return super.newOperation(insn);
+ }
+
+ private static String desc(Class<?> c) {
+ Type t = Type.getType(c);
+ return t.getDescriptor();
+ }
+
+
+
+ static {
+
+ ImmutableMap.Builder<String, ValueHolderIden> builder = ImmutableMap.builder();
+ ImmutableSet.Builder<String> setB = ImmutableSet.builder();
+ for (Class<?> c : ScalarReplacementTypes.CLASSES) {
+ String desc = desc(c);
+ setB.add(desc);
+ String desc2 = desc.substring(1, desc.length() - 1);
+ ValueHolderIden vhi = new ValueHolderIden(c);
+ builder.put(desc, vhi);
+ builder.put(desc2, vhi);
+ }
+ HOLDER_DESCRIPTORS = setB.build();
+ HOLDERS = builder.build();
+ }
+
+ private final static ImmutableMap<String, ValueHolderIden> HOLDERS;
+ public final static ImmutableSet<String> HOLDER_DESCRIPTORS;
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementNode.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementNode.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementNode.java
new file mode 100644
index 0000000..8a38d32
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementNode.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.tree.analysis.Analyzer;
+import org.objectweb.asm.tree.analysis.AnalyzerException;
+import org.objectweb.asm.tree.analysis.BasicValue;
+import org.objectweb.asm.tree.analysis.Frame;
+
+public class ScalarReplacementNode extends MethodNode {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScalarReplacementNode.class);
+
+
+ String[] exceptionsArr;
+ MethodVisitor inner;
+
+ public ScalarReplacementNode(int access, String name, String desc, String signature, String[] exceptions, MethodVisitor inner) {
+ super(access, name, desc, signature, exceptions);
+ this.exceptionsArr = exceptions;
+ this.inner = inner;
+ }
+
+
+ @Override
+ public void visitEnd() {
+ super.visitEnd();
+
+ Analyzer<BasicValue> a = new Analyzer<>(new ReplacingInterpreter());
+ Frame<BasicValue>[] frames;
+ try {
+ frames = a.analyze("Object", this);
+ } catch (AnalyzerException e) {
+ throw new IllegalStateException(e);
+ }
+ TrackingInstructionList list = new TrackingInstructionList(frames, this.instructions);
+ this.instructions = list;
+ InstructionModifier holderV = new InstructionModifier(this.access, this.name, this.desc, this.signature, this.exceptionsArr, list, inner);
+ accept(holderV);
+ }
+
+
+ IntHolder local;
+ public void x(){
+ IntHolder h = new IntHolder();
+ local = h;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementTypes.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementTypes.java
new file mode 100644
index 0000000..a54ca72
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ScalarReplacementTypes.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.Decimal18Holder;
+import org.apache.drill.exec.expr.holders.Decimal28DenseHolder;
+import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38DenseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal9Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.IntervalDayHolder;
+import org.apache.drill.exec.expr.holders.IntervalHolder;
+import org.apache.drill.exec.expr.holders.IntervalYearHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableDateHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal18Holder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28DenseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38DenseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal9Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableIntervalDayHolder;
+import org.apache.drill.exec.expr.holders.NullableIntervalHolder;
+import org.apache.drill.exec.expr.holders.NullableIntervalYearHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampTZHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.TimeStampTZHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+
+import com.google.common.collect.ImmutableSet;
+
+public class ScalarReplacementTypes {
+
+ private ScalarReplacementTypes(){}
+
+ static {
+
+ Class<?>[] classList = {
+ BitHolder.class,
+ IntHolder.class,
+ BigIntHolder.class,
+ Float4Holder.class,
+ Float8Holder.class,
+ Decimal9Holder.class,
+ Decimal18Holder.class,
+ Decimal28SparseHolder.class,
+ Decimal28DenseHolder.class,
+ Decimal38SparseHolder.class,
+ Decimal38DenseHolder.class,
+ IntervalHolder.class,
+ IntervalDayHolder.class,
+ IntervalYearHolder.class,
+ DateHolder.class,
+ TimeHolder.class,
+ TimeStampHolder.class,
+ TimeStampTZHolder.class,
+ VarCharHolder.class,
+ VarBinaryHolder.class,
+ NullableBitHolder.class,
+ NullableIntHolder.class,
+ NullableBigIntHolder.class,
+ NullableFloat4Holder.class,
+ NullableFloat8Holder.class,
+ NullableVarCharHolder.class,
+ NullableVarBinaryHolder.class,
+ NullableDecimal9Holder.class,
+ NullableDecimal18Holder.class,
+ NullableDecimal28SparseHolder.class,
+ NullableDecimal28DenseHolder.class,
+ NullableDecimal38SparseHolder.class,
+ NullableDecimal38DenseHolder.class,
+ NullableIntervalHolder.class,
+ NullableIntervalDayHolder.class,
+ NullableIntervalYearHolder.class,
+ NullableDateHolder.class,
+ NullableTimeHolder.class,
+ NullableTimeStampHolder.class,
+ NullableTimeStampTZHolder.class
+ };
+
+ CLASSES = ImmutableSet.copyOf(classList);
+ }
+
+ public static final ImmutableSet<Class<?>> CLASSES;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/TrackingInstructionList.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/TrackingInstructionList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/TrackingInstructionList.java
new file mode 100644
index 0000000..7cab17c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/TrackingInstructionList.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.tree.AbstractInsnNode;
+import org.objectweb.asm.tree.InsnList;
+import org.objectweb.asm.tree.analysis.Frame;
+
+public class TrackingInstructionList extends InsnList {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TrackingInstructionList.class);
+
+ Frame<?> currentFrame;
+ Frame<?> nextFrame;
+ Frame<?>[] frames;
+ InsnList inner;
+ int index = 0;
+
+
+
+ public TrackingInstructionList(Frame<?>[] frames, InsnList inner) {
+ super();
+
+ this.frames = frames;
+ this.inner = inner;
+ }
+
+ public InsnList getInner(){
+ return inner;
+ }
+
+ @Override
+ public void accept(MethodVisitor mv) {
+ AbstractInsnNode insn = inner.getFirst();
+ while (insn != null) {
+ currentFrame = frames[index];
+ nextFrame = index +1 < frames.length ? frames[index+1] : null;
+ insn.accept(mv);
+
+ insn = insn.getNext();
+ index++;
+ }
+ }
+
+
+ @Override
+ public int size() {
+ return inner.size();
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderIden.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderIden.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderIden.java
new file mode 100644
index 0000000..5099fbd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ValueHolderIden.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.List;
+
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.LocalVariablesSorter;
+
+import com.carrotsearch.hppc.ObjectIntOpenHashMap;
+import com.google.common.collect.Lists;
+
+class ValueHolderIden {
+
+ final ObjectIntOpenHashMap<String> fieldMap;
+ final Type[] types;
+ final String[] names;
+ final Type type;
+
+ public ValueHolderIden(Class<?> c) {
+ Field[] fields = c.getFields();
+
+ List<Field> fldList = Lists.newArrayList();
+ for(Field f : fields){
+ if(!Modifier.isStatic(f.getModifiers())) {
+ fldList.add(f);
+ }
+ }
+ this.type = Type.getType(c);
+ this.types = new Type[fldList.size()];
+ this.names = new String[fldList.size()];
+ fieldMap = new ObjectIntOpenHashMap<String>();
+ int i =0;
+ for(Field f : fldList){
+ types[i] = Type.getType(f.getType());
+ names[i] = f.getName();
+ fieldMap.put(f.getName(), i);
+ i++;
+ }
+ }
+
+ private static void initType(int index, Type t, DirectSorter v){
+ switch(t.getSort()){
+ case Type.BOOLEAN:
+ case Type.BYTE:
+ case Type.CHAR:
+ case Type.SHORT:
+ case Type.INT:
+ v.visitInsn(Opcodes.ICONST_0);
+ v.directVarInsn(Opcodes.ISTORE, index);
+ break;
+ case Type.LONG:
+ v.visitInsn(Opcodes.LCONST_0);
+ v.directVarInsn(Opcodes.LSTORE, index);
+ break;
+ case Type.FLOAT:
+ v.visitInsn(Opcodes.FCONST_0);
+ v.directVarInsn(Opcodes.FSTORE, index);
+ break;
+ case Type.DOUBLE:
+ v.visitInsn(Opcodes.DCONST_0);
+ v.directVarInsn(Opcodes.DSTORE, index);
+ break;
+ case Type.OBJECT:
+ v.visitInsn(Opcodes.ACONST_NULL);
+ v.directVarInsn(Opcodes.ASTORE, index);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public ValueHolderSub getHolderSub(DirectSorter adder) {
+ int first = -1;
+ for (int i = 0; i < types.length; i++) {
+ int varIndex = adder.newLocal(types[i]);
+ if (i == 0) {
+ first = varIndex;
+ }
+ }
+
+ return new ValueHolderSub(first);
+
+ }
+
+ public ValueHolderSub getHolderSubWithDefinedLocals(int first){
+ return new ValueHolderSub(first);
+ }
+
+ private int dup(Type t){
+ return t.getSize() == 1 ? Opcodes.DUP : Opcodes.DUP2;
+ }
+
+ public void transferToLocal(DirectSorter adder, int localVariable){
+ for (int i = 0; i < types.length; i++) {
+ Type t = types[i];
+ if(i + 1 < types.length) adder.visitInsn(dup(t)); // don't dup for last value.
+ adder.visitFieldInsn(Opcodes.GETFIELD, type.getInternalName(), names[i], t.getDescriptor());
+ adder.directVarInsn(t.getOpcode(Opcodes.ISTORE), localVariable+i);
+ }
+ }
+
+
+
+
+ public int createLocalAndTrasfer(DirectSorter adder){
+ int first = 0;
+ for (int i = 0; i < types.length; i++) {
+ Type t = types[i];
+ int varIndex = adder.newLocal(t);
+ if (i == 0) {
+ first = varIndex;
+ }
+ }
+ transferToLocal(adder, first);
+ return first;
+ }
+
+ public class ValueHolderSub {
+ private int first;
+
+ public ValueHolderSub(int first) {
+ assert first != -1 : "Create Holder for sub that doesn't have any fields.";
+ this.first = first;
+ }
+
+ public ValueHolderIden iden(){
+ return ValueHolderIden.this;
+ }
+
+ public void init(DirectSorter mv){
+ for (int i = 0; i < types.length; i++) {
+ initType(first+i, types[i], mv);
+ }
+ }
+ public int size(){
+ return types.length;
+ }
+
+ public int first(){
+ return first;
+ }
+
+ public void updateFirst(int newFirst){
+ this.first = newFirst;
+ }
+
+ private int field(String name, InstructionModifier mv) {
+ if (!fieldMap.containsKey(name)) throw new IllegalArgumentException(String.format("Unknown name '%s' on line %d.", name, mv.lastLineNumber));
+ return fieldMap.lget();
+ }
+
+ public void addInsn(String name, InstructionModifier mv, int opcode) {
+ switch (opcode) {
+ case Opcodes.GETFIELD:
+ addKnownInsn(name, mv, Opcodes.ILOAD);
+ return;
+
+ case Opcodes.PUTFIELD:
+ addKnownInsn(name, mv, Opcodes.ISTORE);
+ }
+ }
+
+ public void transfer(InstructionModifier mv, int newStart){
+ if(first == newStart) return;
+ for(int i =0; i < types.length; i++){
+ mv.directVarInsn(types[i].getOpcode(Opcodes.ILOAD), first + i);
+ mv.directVarInsn(types[i].getOpcode(Opcodes.ISTORE), newStart + i);
+ }
+ this.first = newStart;
+ }
+
+ private void addKnownInsn(String name, InstructionModifier mv, int analogOpcode) {
+ int f = field(name, mv);
+ Type t = types[f];
+ mv.directVarInsn(t.getOpcode(analogOpcode), first + f);
+ }
+
+ public void transferToExternal(DirectSorter adder, String owner, String name, String desc){
+
+ // create a new object and assign it to the desired field.
+ adder.visitTypeInsn(Opcodes.NEW, type.getInternalName());
+ adder.visitInsn(dup(type));
+ adder.visitMethodInsn(Opcodes.INVOKESPECIAL, type.getInternalName(), "<init>", "()V");
+
+ // now we need to set all of the values of this new object.
+ for (int i = 0; i < types.length; i++) {
+ Type t = types[i];
+
+ // dup the object where we are putting the field.
+ adder.visitInsn(dup(type)); // dup for every as we want to save in place at end.
+ adder.directVarInsn(t.getOpcode(Opcodes.ILOAD), first+i);
+ adder.visitFieldInsn(Opcodes.PUTFIELD, type.getInternalName(), names[i], t.getDescriptor());
+ }
+
+ // lastly we save it to the desired field.
+ adder.visitFieldInsn(Opcodes.PUTFIELD, owner, name, desc);
+
+ }
+
+ }
+
+
+}
\ No newline at end of file
[28/47] git commit: Fix test case failures in TestParquetWriter
Posted by ja...@apache.org.
Fix test case failures in TestParquetWriter
* Use different table names for different test cases.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b5bbed84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b5bbed84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b5bbed84
Branch: refs/heads/master
Commit: b5bbed84bccdff39f1b13c4d529fa8dbf8652220
Parents: f1a46e6
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Mon Aug 25 23:48:18 2014 +0530
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Mon Aug 25 23:48:18 2014 +0530
----------------------------------------------------------------------
.../drill/exec/physical/impl/writer/TestParquetWriter.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b5bbed84/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index d399fa9..b4fd09d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -92,7 +92,7 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
public void testTPCHReadWrite1() throws Exception {
String inputTable = "cp.`tpch/lineitem.parquet`";
- runTestAndValidate("*", "*", inputTable, "lineitem_parquet");
+ runTestAndValidate("*", "*", inputTable, "lineitem_parquet_all");
}
@Test
@@ -102,7 +102,7 @@ public class TestParquetWriter extends BaseTestQuery {
String validationSelection = "L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, " +
"L_RETURNFLAG, L_LINESTATUS, L_SHIPDATE,L_COMMITDATE ,L_RECEIPTDATE, L_SHIPINSTRUCT, L_SHIPMODE, L_COMMENT";
String inputTable = "cp.`tpch/lineitem.parquet`";
- runTestAndValidate(selection, validationSelection, inputTable, "lineitem_parquet");
+ runTestAndValidate(selection, validationSelection, inputTable, "lineitem_parquet_converted");
}
@Test
[30/47] git commit: Fix initCap function to set outbound position
correctly.
Posted by ja...@apache.org.
Fix initCap function to set outbound position correctly.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7fc8d6d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7fc8d6d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7fc8d6d4
Branch: refs/heads/master
Commit: 7fc8d6d4dfee9fd37327efa205145e75eaf2cde1
Parents: 9187035
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Aug 25 22:30:37 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Aug 25 22:32:16 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7fc8d6d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
index 68f7ca4..d429a28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
@@ -32,8 +32,8 @@ public class StringFunctionHelpers {
// white space is treated as everything else.
public static void initCap(int start, int end, DrillBuf inBuf, DrillBuf outBuf) {
boolean capNext = true;
-
- for (int id = start; id < end; id++) {
+ int out = 0;
+ for (int id = start; id < end; id++; out++) {
byte currentByte = inBuf.getByte(id);
// 'A - Z' : 0x41 - 0x5A
@@ -61,7 +61,7 @@ public class StringFunctionHelpers {
}
}
- outBuf.setByte(id, currentByte);
+ outBuf.setByte(out, currentByte);
} // end of for_loop
}
[10/47] git commit: Update TestViewSupport so that it doesn't have a
null column.
Posted by ja...@apache.org.
Update TestViewSupport so that it doesn't have a null column.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f1136ab9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f1136ab9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f1136ab9
Branch: refs/heads/master
Commit: f1136ab94c8cf64218e7ad4952c5d0b2cf6c3434
Parents: 3529650
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Aug 24 12:02:30 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 12:02:30 2014 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/drill/exec/sql/TestViewSupport.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f1136ab9/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
index b745baa..bfd3f18 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
@@ -26,8 +26,8 @@ public class TestViewSupport extends BaseTestQuery{
@Test
public void referToSchemaInsideAndOutsideView() throws Exception {
String use = "use dfs_test.tmp;";
- String selectInto = "create table monkey as select c_custkey, c_regionkey from cp.`tpch/customer.parquet`";
- String createView = "create or replace view myMonkeyView as select c_custkey, c_regionkey from monkey";
+ String selectInto = "create table monkey as select c_custkey, c_nationkey from cp.`tpch/customer.parquet`";
+ String createView = "create or replace view myMonkeyView as select c_custkey, c_nationkey from monkey";
String selectInside = "select * from myMonkeyView;";
String use2 = "use cp;";
String selectOutside = "select * from dfs_test.tmp.myMonkeyView;";
[45/47] git commit: DRILL-1353: Fix decimal regressions
Posted by ja...@apache.org.
DRILL-1353: Fix decimal regressions
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d8a770fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d8a770fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d8a770fd
Branch: refs/heads/master
Commit: d8a770fd98be1c1cd4629da98f7dfcb5a775e0ba
Parents: 528308c
Author: Mehant Baid <me...@gmail.com>
Authored: Wed Aug 27 14:27:31 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 18:44:02 2014 -0700
----------------------------------------------------------------------
.../templates/Decimal/CastDecimalSimilar.java | 2 +-
.../exec/expr/fn/impl/ByteFunctionHelpers.java | 52 ++++++++++++++++++--
.../drill/exec/expr/fn/impl/DrillByteArray.java | 6 ++-
3 files changed, 52 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d8a770fd/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
index 8c46b49..cd2c390 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
@@ -83,7 +83,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
if (in.scale != out.scale) {
org.apache.drill.exec.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale);
}
- out.setSign(sign, in.start, in.buffer);
+ out.setSign(sign, out.start, out.buffer);
}
}
</#if> <#-- type.major -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d8a770fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
index b524139..cbb87bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
@@ -26,6 +26,17 @@ import org.apache.drill.exec.util.DecimalUtility;
public class ByteFunctionHelpers {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
+ /**
+ * Helper function to check for equality of bytes in two DrillBuffers
+ *
+ * @param laddr start address of the DrillBuf
+ * @param lStart start offset in the buffer
+ * @param lEnd end offset in the buffer
+ * @param raddr start address of the DrillBuf
+ * @param rStart start offset in the buffer
+ * @param rEnd end offset in the buffer
+ * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+ */
public static final int equal(final long laddr, int lStart, int lEnd, final long raddr, int rStart,
final int rEnd) {
@@ -59,6 +70,17 @@ public class ByteFunctionHelpers {
}
}
+ /**
+ * Helper function to compare a set of bytes in two DrillBuffers
+ *
+ * @param laddr start address of the DrillBuf
+ * @param lStart start offset in the buffer
+ * @param lEnd end offset in the buffer
+ * @param raddr start address of the DrillBuf
+ * @param rStart start offset in the buffer
+ * @param rEnd end offset in the buffer
+ * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+ */
public static final int compare(final long laddr, int lStart, int lEnd, final long raddr, int rStart, final int rEnd) {
int lLen = lEnd - lStart;
int rLen = rEnd - rStart;
@@ -93,6 +115,17 @@ public class ByteFunctionHelpers {
}
+ /**
+ * Helper function to compare a set of bytes in DrillBuf to a ByteArray.
+ *
+ * @param laddr start address of the DrillBuf
+ * @param lStart start offset in the buffer
+ * @param lEnd end offset in the buffer
+ * @param right second input to be compared
+ * @param rStart start offset in the byte array
+ * @param rEnd end offset in the byte array
+ * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+ */
public static final int compare(final long laddr, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
int lLen = lEnd - lStart;
int rLen = rEnd - rStart;
@@ -117,11 +150,20 @@ public class ByteFunctionHelpers {
return lLen > rLen ? 1 : -1;
}
- // Get the big endian integer
+
+ /*
+ * Following are helper functions to interact with sparse decimal represented in a byte array.
+ */
+
+ // Get the integer ignore the sign
public static int getInteger(byte[] b, int index) {
+ return getInteger(b, index, true);
+ }
+ // Get the integer, ignore the sign
+ public static int getInteger(byte[] b, int index, boolean ignoreSign) {
int startIndex = index * DecimalUtility.integerSize;
- if (index == 0) {
+ if (index == 0 && ignoreSign == true) {
return (b[startIndex + 3] & 0xFF) |
(b[startIndex + 2] & 0xFF) << 8 |
(b[startIndex + 1] & 0xFF) << 16 |
@@ -133,9 +175,9 @@ public class ByteFunctionHelpers {
(b[startIndex + 1] & 0xFF) << 16 |
(b[startIndex] & 0xFF) << 24);
- }
+ }
- // Set the big endian bytes for the input integer
+ // Set integer in the byte array
public static void setInteger(byte[] b, int index, int value) {
int startIndex = index * DecimalUtility.integerSize;
b[startIndex] = (byte) ((value >> 24) & 0xFF);
@@ -156,6 +198,6 @@ public class ByteFunctionHelpers {
// Get the sign
public static boolean getSign(byte[] b) {
- return ((b[0] & 0x80) > 0);
+ return ((getInteger(b, 0, false) & 0x80000000) != 0);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d8a770fd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
index 507db67..d1c67ef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillByteArray.java
@@ -17,8 +17,10 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import io.netty.buffer.ByteBuf;
-
+/**
+ * Thin wrapper around byte array. This class is used by aggregate functions which
+ * consume decimal, variable width vectors as inputs.
+ */
public class DrillByteArray {
private byte[] bytes;
private int length;
[04/47] git commit: DRILL-1281: Shaded hbase jar
Posted by ja...@apache.org.
DRILL-1281: Shaded hbase jar
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a0a8256c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a0a8256c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a0a8256c
Branch: refs/heads/master
Commit: a0a8256c0f054e3754e9da18411859da27f9a1ca
Parents: c3b15af
Author: Steven Phillips <sp...@maprtech.com>
Authored: Mon Aug 18 14:16:26 2014 -0700
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Mon Aug 18 14:46:48 2014 -0700
----------------------------------------------------------------------
contrib/hbase-shaded/pom.xml | 373 ++++++++++++++++++++++++++++
contrib/pom.xml | 1 +
distribution/pom.xml | 217 +++++++++++++---
distribution/src/assemble/bin-mapr.xml | 180 ++++++++++++++
distribution/src/assemble/bin.xml | 1 +
pom.xml | 15 ++
6 files changed, 753 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a0a8256c/contrib/hbase-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/hbase-shaded/pom.xml b/contrib/hbase-shaded/pom.xml
new file mode 100644
index 0000000..cc61291
--- /dev/null
+++ b/contrib/hbase-shaded/pom.xml
@@ -0,0 +1,373 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+ xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.drill.contrib</groupId>
+ <artifactId>drill-contrib-parent</artifactId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>drill-hbase-shaded</artifactId>
+ <packaging>jar</packaging>
+ <name>contrib/hbase-shaded</name>
+
+ <properties>
+ <hbase.TestSuite>**/HBaseTestsSuite.class</hbase.TestSuite>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>default-hadoop</id>
+ <activation>
+ <property>
+ <name>!alt-hadoop</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.11</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>libthrift</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jruby-complete</artifactId>
+ <groupId>org.jruby</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- contrary to other Drill modules, this HBase storage module
+ compiles against hadoop 1.x to allow running a mini HBase
+ cluster to run the unit tests.
+ This will change once we move to HBase 0.98 or later.
+ -->
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>1.2.1</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hbase:hbase</include>
+ <include>org.apache.hadoop:hadoop-core</include>
+ </includes>
+ </artifactSet>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.hadoop.net</pattern>
+ <shadedPattern>org.apache.hadoop1.net</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>mapr</id>
+ <properties>
+ <alt-hadoop>mapr</alt-hadoop>
+ <rat.excludeSubprojects>true</rat.excludeSubprojects>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.17-mapr-1405-m7-4.0.0-FCS</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>libthrift</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jruby-complete</artifactId>
+ <groupId>org.jruby</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-core</artifactId>
+ <groupId>org.apache</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ <version>1.0.3-mapr-3.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <classifier>tests</classifier>
+ <version>0.94.13-mapr-1401-m7-3.0.2</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters>
+ <filter>
+ <artifact>org.apache.hadoop:hadoop-core</artifact>
+ <excludes>
+ <exclude>org.apache.hadoop.fs.*</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hbase:hbase</include>
+ <include>org.apache.hadoop:hadoop-core</include>
+ </includes>
+ </artifactSet>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.hadoop.net</pattern>
+ <shadedPattern>org.apache.hadoop1.net</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <includes>
+ <include>${hbase.TestSuite}</include>
+ </includes>
+ <systemProperties>
+ <property>
+ <name>hbase.test.root</name>
+ <value>${project.build.directory}/data</value>
+ </property>
+ <property>
+ <name>logback.log.dir</name>
+ <value>${project.build.directory}/surefire-reports</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hbase:hbase</include>
+ <include>org.apache.hadoop:hadoop-core</include>
+ </includes>
+ </artifactSet>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.hadoop.net</pattern>
+ <shadedPattern>org.apache.hadoop1.net</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>unpack</id>
+ <phase>package</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.drill.contrib</groupId>
+ <artifactId>drill-hbase-shaded</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}/classes</outputDirectory>
+ <includes>**/**</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a0a8256c/contrib/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/pom.xml b/contrib/pom.xml
index de7709d..0c979e9 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -32,6 +32,7 @@
</dependencies>
<modules>
+ <module>hbase-shaded</module>
<module>storage-hbase</module>
<module>storage-hive</module>
<module>sqlline</module>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a0a8256c/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 1429902..0d493ce 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -62,17 +62,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.drill.contrib</groupId>
- <artifactId>drill-storage-hbase</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.drill.contrib.storage-hive</groupId>
<artifactId>drill-storage-hive-core</artifactId>
<version>${project.version}</version>
@@ -103,33 +92,193 @@
</configuration>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>distro-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>src/assemble/bin.xml</descriptor>
- </descriptors>
- <appendAssemblyId>false</appendAssemblyId>
- <finalName>apache-drill-${project.version}</finalName>
- <tarLongFileMode>gnu</tarLongFileMode>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
-
-
-
<profiles>
<profile>
+ <id>default-hadoop</id>
+ <activation>
+ <property>
+ <name>!alt-hadoop</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.drill.contrib</groupId>
+ <artifactId>drill-storage-hbase</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.contrib</groupId>
+ <artifactId>drill-hbase-shaded</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>distro-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assemble/bin.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <finalName>apache-drill-${project.version}</finalName>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>mapr</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.drill.contrib</groupId>
+ <artifactId>drill-storage-hbase</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.mapr.hadoop</groupId>
+ <artifactId>maprfs</artifactId>
+ </dependency>
+ <!--
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.17-mapr-1405-m7-4.0.0-FCS</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>libthrift</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jruby-complete</artifactId>
+ <groupId>org.jruby</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.mapr.hadoop</groupId>
+ <artifactId>maprfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.mapr.fs</groupId>
+ <artifactId>libprotodefs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.mapr.fs</groupId>
+ <artifactId>mapr-hbase</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ -->
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>distro-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assemble/bin-mapr.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <finalName>apache-drill-${project.version}</finalName>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>rpm</id>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a0a8256c/distribution/src/assemble/bin-mapr.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin-mapr.xml b/distribution/src/assemble/bin-mapr.xml
new file mode 100644
index 0000000..ad7bb26
--- /dev/null
+++ b/distribution/src/assemble/bin-mapr.xml
@@ -0,0 +1,180 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>binary-release</id>
+ <formats>
+ <format>tar.gz</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <moduleSets>
+
+ </moduleSets>
+ <dependencySets>
+<!--
+ <dependencySet>
+ <includes>
+ <include>org.apache.drill.exec:drill-jdbc-all:jar</include>
+ </includes>
+ <outputDirectory>jdbc-driver</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
+-->
+ <dependencySet>
+ <!-- Now, select which projects to include in this module-set. -->
+ <includes>
+ <include>org.apache.drill.exec:drill-jdbc:jar</include>
+ <include>org.apache.drill:drill-protocol:jar:rebuffed</include>
+ <include>org.apache.drill:drill-common:jar:rebuffed</include>
+ <include>org.apache.drill.exec:drill-java-exec:jar:rebuffed</include>
+ <include>org.apache.drill.contrib.storage-hive:drill-storage-hive-core</include>
+ <include>org.apache.drill.contrib.storage-hive:drill-hive-exec-shaded</include>
+ <include>org.apache.drill.contrib.data:tpch-sample-data:jar</include>
+ <include>org.apache.drill.contrib:drill-storage-hbase</include>
+ </includes>
+ <excludes>
+ <exclude>org.apache.drill.contrib.storage-hive:drill-storage-hive-core:jar:tests</exclude>
+ </excludes>
+ <outputDirectory>jars</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
+
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <unpack>false</unpack>
+ <useProjectArtifact>false</useProjectArtifact>
+ <excludes>
+ <exclude>org.apache.zookeeper</exclude>
+ <exclude>org.apache.drill</exclude>
+ <exclude>org.apache.drill.exec</exclude>
+ <exclude>org.apache.drill.contrib</exclude>
+ <exclude>org.apache.drill.contrib.storage-hive</exclude>
+ <exclude>org.hsqldb:hsqldb</exclude>
+ <!-- exclude or sqlline has problems -->
+ <exclude>hsqldb:hsqldb</exclude>
+ <!-- exclude or sqlline has problems -->
+ </excludes>
+ <scope>test</scope>
+ </dependencySet>
+
+ <dependencySet>
+ <outputDirectory>extlib</outputDirectory>
+ <unpack>false</unpack>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>org.apache.zookeeper:zookeeper</include>
+ </includes>
+ <scope>runtime</scope>
+ </dependencySet>
+
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <unpack>false</unpack>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>com.google.protobuf:protobuf-java:jar:2.5</include>
+ </includes>
+ <scope>test</scope>
+ </dependencySet>
+
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>../sample-data</directory>
+ <outputDirectory>sample-data</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <source>../KEYS</source>
+ <outputDirectory/>
+ </file>
+ <file>
+ <source>../LICENSE</source>
+ <outputDirectory/>
+ </file>
+ <file>
+ <source>../NOTICE</source>
+ <outputDirectory/>
+ </file>
+ <file>
+ <source>../git.properties</source>
+ <outputDirectory/>
+ </file>
+ <file>
+ <source>src/resources/runbit</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/hadoop-excludes.txt</source>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drillbit.sh</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drill-config.sh</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/sqlline</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/sqlline.bat</source>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/submit_plan</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drill_dumpcat</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drill-override.conf</source>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/logback.xml</source>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drill-env.sh</source>
+ <fileMode>0755</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ <file>
+ <source>src/resources/drill-override-example.conf</source>
+ <outputDirectory>conf</outputDirectory>
+ </file>
+ </files>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a0a8256c/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index 6d7fb6b..68b58bf 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -49,6 +49,7 @@
<include>org.apache.drill.contrib.storage-hive:drill-hive-exec-shaded</include>
<include>org.apache.drill.contrib.data:tpch-sample-data:jar</include>
<include>org.apache.drill.contrib:drill-storage-hbase</include>
+ <include>org.apache.drill.contrib:drill-hbase-shaded</include>
</includes>
<excludes>
<exclude>org.apache.drill.contrib.storage-hive:drill-storage-hive-core:jar:tests</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a0a8256c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 457ef3e..f04d5b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -774,6 +774,21 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>com.mapr.fs</groupId>
+ <artifactId>libprotodefs</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>com.mapr.fs</groupId>
+ <artifactId>mapr-hbase</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>com.mapr.hadoop</groupId>
+ <artifactId>maprfs</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<version>2.11.0</version>
[40/47] git commit: DRILL-1352: C++ Client. Update the decoding of
nullable value vectors to read a byte for every nullable bit.
Posted by ja...@apache.org.
DRILL-1352: C++ Client. Update the decoding of nullable value vectors to read a byte for every nullable bit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cc25504d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cc25504d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cc25504d
Branch: refs/heads/master
Commit: cc25504d1bfeb7ac7bc99a6c3ce5285d30e13697
Parents: ceb58c4
Author: Parth Chandra <pc...@maprtech.com>
Authored: Wed Aug 27 18:07:35 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 18:36:30 2014 -0700
----------------------------------------------------------------------
.../native/client/src/include/drill/common.hpp | 2 +-
.../src/include/drill/protobuf/Types.pb.h | 5 +-
.../client/src/include/drill/recordBatch.hpp | 10 +-
.../native/client/src/protobuf/BitControl.pb.cc | 305 ++++++++++++++++++-
.../native/client/src/protobuf/BitControl.pb.h | 194 +++++++++++-
contrib/native/client/src/protobuf/Types.pb.cc | 10 +-
.../client/src/protobuf/UserBitShared.pb.cc | 28 +-
.../client/src/protobuf/UserBitShared.pb.h | 7 +-
pom.xml | 1 +
9 files changed, 517 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/include/drill/common.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/include/drill/common.hpp b/contrib/native/client/src/include/drill/common.hpp
index dfb04e8..151d698 100644
--- a/contrib/native/client/src/include/drill/common.hpp
+++ b/contrib/native/client/src/include/drill/common.hpp
@@ -33,7 +33,7 @@
#include <vector>
#include <boost/shared_ptr.hpp>
-#define DRILL_RPC_VERSION 1
+#define DRILL_RPC_VERSION 2
#define LENGTH_PREFIX_MAX_LENGTH 5
#define LEN_PREFIX_BUFLEN LENGTH_PREFIX_MAX_LENGTH
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/include/drill/protobuf/Types.pb.h
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/include/drill/protobuf/Types.pb.h b/contrib/native/client/src/include/drill/protobuf/Types.pb.h
index 8534153..b26177c 100644
--- a/contrib/native/client/src/include/drill/protobuf/Types.pb.h
+++ b/contrib/native/client/src/include/drill/protobuf/Types.pb.h
@@ -72,11 +72,12 @@ enum MinorType {
DM_UNKNOWN = 37,
INTERVALYEAR = 38,
INTERVALDAY = 39,
- LIST = 40
+ LIST = 40,
+ GENERIC_OBJECT = 41
};
bool MinorType_IsValid(int value);
const MinorType MinorType_MIN = LATE;
-const MinorType MinorType_MAX = LIST;
+const MinorType MinorType_MAX = GENERIC_OBJECT;
const int MinorType_ARRAYSIZE = MinorType_MAX + 1;
const ::google::protobuf::EnumDescriptor* MinorType_descriptor();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/include/drill/recordBatch.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/include/drill/recordBatch.hpp b/contrib/native/client/src/include/drill/recordBatch.hpp
index 9a3df2b..61db88c 100644
--- a/contrib/native/client/src/include/drill/recordBatch.hpp
+++ b/contrib/native/client/src/include/drill/recordBatch.hpp
@@ -386,7 +386,7 @@ template <typename VALUE_TYPE>
{
public:
NullableValueVectorFixed(SlicedByteBuf *b, size_t rowCount):ValueVectorBase(b, rowCount){
- size_t offsetEnd = (size_t)ceil(rowCount/8.0);
+ size_t offsetEnd = (size_t)rowCount;
this->m_pBitmap= new SlicedByteBuf(*b, 0, offsetEnd);
this->m_pData= new SlicedByteBuf(*b, offsetEnd, b->getLength());
// TODO: testing boundary case(null columns)
@@ -399,7 +399,7 @@ template <typename VALUE_TYPE>
// test whether the value is null in the position index
bool isNull(size_t index) const {
- return (m_pBitmap->getBit(index)==0);
+ return (m_pBitmap->getByte(index)==0);
}
VALUE_TYPE get(size_t index) const {
@@ -584,14 +584,14 @@ template <class VALUEHOLDER_CLASS_TYPE, class VALUE_VECTOR_TYPE>
public:
NullableValueVectorTyped(SlicedByteBuf *b, size_t rowCount):ValueVectorBase(b, rowCount){
- size_t offsetEnd = (size_t)ceil(rowCount/8.0);
+ size_t offsetEnd = (size_t)rowCount;
this->m_pBitmap= new SlicedByteBuf(*b, 0, offsetEnd);
this->m_pData= new SlicedByteBuf(*b, offsetEnd, b->getLength()-offsetEnd);
this->m_pVector= new VALUE_VECTOR_TYPE(m_pData, rowCount);
}
// Specialized for Decimal Types
NullableValueVectorTyped(SlicedByteBuf *b, size_t rowCount, int32_t scale):ValueVectorBase(b, rowCount){
- size_t offsetEnd = (size_t)ceil(rowCount/8.0);
+ size_t offsetEnd = (size_t)rowCount;
this->m_pBitmap= new SlicedByteBuf(*b, 0, offsetEnd);
this->m_pData= new SlicedByteBuf(*b, offsetEnd, b->getLength()-offsetEnd);
this->m_pVector= new VALUE_VECTOR_TYPE(m_pData, rowCount, scale);
@@ -604,7 +604,7 @@ template <class VALUEHOLDER_CLASS_TYPE, class VALUE_VECTOR_TYPE>
}
bool isNull(size_t index) const{
- return (m_pBitmap->getBit(index)==0);
+ return (m_pBitmap->getByte(index)==0);
}
VALUEHOLDER_CLASS_TYPE get(size_t index) const {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/protobuf/BitControl.pb.cc
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/protobuf/BitControl.pb.cc b/contrib/native/client/src/protobuf/BitControl.pb.cc
index 4f8749c..a762386 100644
--- a/contrib/native/client/src/protobuf/BitControl.pb.cc
+++ b/contrib/native/client/src/protobuf/BitControl.pb.cc
@@ -37,6 +37,9 @@ const ::google::protobuf::internal::GeneratedMessageReflection*
const ::google::protobuf::Descriptor* WorkQueueStatus_descriptor_ = NULL;
const ::google::protobuf::internal::GeneratedMessageReflection*
WorkQueueStatus_reflection_ = NULL;
+const ::google::protobuf::Descriptor* FinishedReceiver_descriptor_ = NULL;
+const ::google::protobuf::internal::GeneratedMessageReflection*
+ FinishedReceiver_reflection_ = NULL;
const ::google::protobuf::EnumDescriptor* RpcType_descriptor_ = NULL;
} // namespace
@@ -142,6 +145,22 @@ void protobuf_AssignDesc_BitControl_2eproto() {
::google::protobuf::DescriptorPool::generated_pool(),
::google::protobuf::MessageFactory::generated_factory(),
sizeof(WorkQueueStatus));
+ FinishedReceiver_descriptor_ = file->message_type(5);
+ static const int FinishedReceiver_offsets_[2] = {
+ GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, receiver_),
+ GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, sender_),
+ };
+ FinishedReceiver_reflection_ =
+ new ::google::protobuf::internal::GeneratedMessageReflection(
+ FinishedReceiver_descriptor_,
+ FinishedReceiver::default_instance_,
+ FinishedReceiver_offsets_,
+ GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, _has_bits_[0]),
+ GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, _unknown_fields_),
+ -1,
+ ::google::protobuf::DescriptorPool::generated_pool(),
+ ::google::protobuf::MessageFactory::generated_factory(),
+ sizeof(FinishedReceiver));
RpcType_descriptor_ = file->enum_type(0);
}
@@ -165,6 +184,8 @@ void protobuf_RegisterTypes(const ::std::string&) {
PlanFragment_descriptor_, &PlanFragment::default_instance());
::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
WorkQueueStatus_descriptor_, &WorkQueueStatus::default_instance());
+ ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage(
+ FinishedReceiver_descriptor_, &FinishedReceiver::default_instance());
}
} // namespace
@@ -180,6 +201,8 @@ void protobuf_ShutdownFile_BitControl_2eproto() {
delete PlanFragment_reflection_;
delete WorkQueueStatus::default_instance_;
delete WorkQueueStatus_reflection_;
+ delete FinishedReceiver::default_instance_;
+ delete FinishedReceiver_reflection_;
}
void protobuf_AddDesc_BitControl_2eproto() {
@@ -216,15 +239,18 @@ void protobuf_AddDesc_BitControl_2eproto() {
"e_zone\030\020 \001(\005\022\024\n\014options_json\030\021 \001(\t\"f\n\017Wo"
"rkQueueStatus\022(\n\010endpoint\030\001 \001(\0132\026.exec.D"
"rillbitEndpoint\022\024\n\014queue_length\030\002 \001(\005\022\023\n"
- "\013report_time\030\003 \001(\003*\207\002\n\007RpcType\022\r\n\tHANDSH"
- "AKE\020\000\022\007\n\003ACK\020\001\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_INIAT"
- "ILIZE_FRAGMENT\020\003\022\027\n\023REQ_CANCEL_FRAGMENT\020"
- "\006\022\027\n\023REQ_FRAGMENT_STATUS\020\007\022\022\n\016REQ_BIT_ST"
- "ATUS\020\010\022\024\n\020REQ_QUERY_STATUS\020\t\022\030\n\024RESP_FRA"
- "GMENT_HANDLE\020\n\022\030\n\024RESP_FRAGMENT_STATUS\020\013"
- "\022\023\n\017RESP_BIT_STATUS\020\014\022\025\n\021RESP_QUERY_STAT"
- "US\020\rB+\n\033org.apache.drill.exec.protoB\nBit"
- "ControlH\001", 1289);
+ "\013report_time\030\003 \001(\003\"h\n\020FinishedReceiver\022*"
+ "\n\010receiver\030\001 \001(\0132\030.exec.bit.FragmentHand"
+ "le\022(\n\006sender\030\002 \001(\0132\030.exec.bit.FragmentHa"
+ "ndle*\242\002\n\007RpcType\022\r\n\tHANDSHAKE\020\000\022\007\n\003ACK\020\001"
+ "\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_INIATILIZE_FRAGMENT"
+ "\020\003\022\027\n\023REQ_CANCEL_FRAGMENT\020\006\022\031\n\025REQ_RECEI"
+ "VER_FINISHED\020\007\022\027\n\023REQ_FRAGMENT_STATUS\020\010\022"
+ "\022\n\016REQ_BIT_STATUS\020\t\022\024\n\020REQ_QUERY_STATUS\020"
+ "\n\022\030\n\024RESP_FRAGMENT_HANDLE\020\013\022\030\n\024RESP_FRAG"
+ "MENT_STATUS\020\014\022\023\n\017RESP_BIT_STATUS\020\r\022\025\n\021RE"
+ "SP_QUERY_STATUS\020\016B+\n\033org.apache.drill.ex"
+ "ec.protoB\nBitControlH\001", 1422);
::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
"BitControl.proto", &protobuf_RegisterTypes);
BitControlHandshake::default_instance_ = new BitControlHandshake();
@@ -232,11 +258,13 @@ void protobuf_AddDesc_BitControl_2eproto() {
FragmentStatus::default_instance_ = new FragmentStatus();
PlanFragment::default_instance_ = new PlanFragment();
WorkQueueStatus::default_instance_ = new WorkQueueStatus();
+ FinishedReceiver::default_instance_ = new FinishedReceiver();
BitControlHandshake::default_instance_->InitAsDefaultInstance();
BitStatus::default_instance_->InitAsDefaultInstance();
FragmentStatus::default_instance_->InitAsDefaultInstance();
PlanFragment::default_instance_->InitAsDefaultInstance();
WorkQueueStatus::default_instance_->InitAsDefaultInstance();
+ FinishedReceiver::default_instance_->InitAsDefaultInstance();
::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_BitControl_2eproto);
}
@@ -264,6 +292,7 @@ bool RpcType_IsValid(int value) {
case 11:
case 12:
case 13:
+ case 14:
return true;
default:
return false;
@@ -2142,6 +2171,264 @@ void WorkQueueStatus::Swap(WorkQueueStatus* other) {
}
+// ===================================================================
+
+#ifndef _MSC_VER
+const int FinishedReceiver::kReceiverFieldNumber;
+const int FinishedReceiver::kSenderFieldNumber;
+#endif // !_MSC_VER
+
+FinishedReceiver::FinishedReceiver()
+ : ::google::protobuf::Message() {
+ SharedCtor();
+}
+
+void FinishedReceiver::InitAsDefaultInstance() {
+ receiver_ = const_cast< ::exec::bit::FragmentHandle*>(&::exec::bit::FragmentHandle::default_instance());
+ sender_ = const_cast< ::exec::bit::FragmentHandle*>(&::exec::bit::FragmentHandle::default_instance());
+}
+
+FinishedReceiver::FinishedReceiver(const FinishedReceiver& from)
+ : ::google::protobuf::Message() {
+ SharedCtor();
+ MergeFrom(from);
+}
+
+void FinishedReceiver::SharedCtor() {
+ _cached_size_ = 0;
+ receiver_ = NULL;
+ sender_ = NULL;
+ ::memset(_has_bits_, 0, sizeof(_has_bits_));
+}
+
+FinishedReceiver::~FinishedReceiver() {
+ SharedDtor();
+}
+
+void FinishedReceiver::SharedDtor() {
+ if (this != default_instance_) {
+ delete receiver_;
+ delete sender_;
+ }
+}
+
+void FinishedReceiver::SetCachedSize(int size) const {
+ GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+ _cached_size_ = size;
+ GOOGLE_SAFE_CONCURRENT_WRITES_END();
+}
+const ::google::protobuf::Descriptor* FinishedReceiver::descriptor() {
+ protobuf_AssignDescriptorsOnce();
+ return FinishedReceiver_descriptor_;
+}
+
+const FinishedReceiver& FinishedReceiver::default_instance() {
+ if (default_instance_ == NULL) protobuf_AddDesc_BitControl_2eproto();
+ return *default_instance_;
+}
+
+FinishedReceiver* FinishedReceiver::default_instance_ = NULL;
+
+FinishedReceiver* FinishedReceiver::New() const {
+ return new FinishedReceiver;
+}
+
+void FinishedReceiver::Clear() {
+ if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) {
+ if (has_receiver()) {
+ if (receiver_ != NULL) receiver_->::exec::bit::FragmentHandle::Clear();
+ }
+ if (has_sender()) {
+ if (sender_ != NULL) sender_->::exec::bit::FragmentHandle::Clear();
+ }
+ }
+ ::memset(_has_bits_, 0, sizeof(_has_bits_));
+ mutable_unknown_fields()->Clear();
+}
+
+bool FinishedReceiver::MergePartialFromCodedStream(
+ ::google::protobuf::io::CodedInputStream* input) {
+#define DO_(EXPRESSION) if (!(EXPRESSION)) return false
+ ::google::protobuf::uint32 tag;
+ while ((tag = input->ReadTag()) != 0) {
+ switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) {
+ // optional .exec.bit.FragmentHandle receiver = 1;
+ case 1: {
+ if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+ ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED) {
+ DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+ input, mutable_receiver()));
+ } else {
+ goto handle_uninterpreted;
+ }
+ if (input->ExpectTag(18)) goto parse_sender;
+ break;
+ }
+
+ // optional .exec.bit.FragmentHandle sender = 2;
+ case 2: {
+ if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+ ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED) {
+ parse_sender:
+ DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual(
+ input, mutable_sender()));
+ } else {
+ goto handle_uninterpreted;
+ }
+ if (input->ExpectAtEnd()) return true;
+ break;
+ }
+
+ default: {
+ handle_uninterpreted:
+ if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+ ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) {
+ return true;
+ }
+ DO_(::google::protobuf::internal::WireFormat::SkipField(
+ input, tag, mutable_unknown_fields()));
+ break;
+ }
+ }
+ }
+ return true;
+#undef DO_
+}
+
+void FinishedReceiver::SerializeWithCachedSizes(
+ ::google::protobuf::io::CodedOutputStream* output) const {
+ // optional .exec.bit.FragmentHandle receiver = 1;
+ if (has_receiver()) {
+ ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+ 1, this->receiver(), output);
+ }
+
+ // optional .exec.bit.FragmentHandle sender = 2;
+ if (has_sender()) {
+ ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
+ 2, this->sender(), output);
+ }
+
+ if (!unknown_fields().empty()) {
+ ::google::protobuf::internal::WireFormat::SerializeUnknownFields(
+ unknown_fields(), output);
+ }
+}
+
+::google::protobuf::uint8* FinishedReceiver::SerializeWithCachedSizesToArray(
+ ::google::protobuf::uint8* target) const {
+ // optional .exec.bit.FragmentHandle receiver = 1;
+ if (has_receiver()) {
+ target = ::google::protobuf::internal::WireFormatLite::
+ WriteMessageNoVirtualToArray(
+ 1, this->receiver(), target);
+ }
+
+ // optional .exec.bit.FragmentHandle sender = 2;
+ if (has_sender()) {
+ target = ::google::protobuf::internal::WireFormatLite::
+ WriteMessageNoVirtualToArray(
+ 2, this->sender(), target);
+ }
+
+ if (!unknown_fields().empty()) {
+ target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
+ unknown_fields(), target);
+ }
+ return target;
+}
+
+int FinishedReceiver::ByteSize() const {
+ int total_size = 0;
+
+ if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) {
+ // optional .exec.bit.FragmentHandle receiver = 1;
+ if (has_receiver()) {
+ total_size += 1 +
+ ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+ this->receiver());
+ }
+
+ // optional .exec.bit.FragmentHandle sender = 2;
+ if (has_sender()) {
+ total_size += 1 +
+ ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual(
+ this->sender());
+ }
+
+ }
+ if (!unknown_fields().empty()) {
+ total_size +=
+ ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize(
+ unknown_fields());
+ }
+ GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN();
+ _cached_size_ = total_size;
+ GOOGLE_SAFE_CONCURRENT_WRITES_END();
+ return total_size;
+}
+
+void FinishedReceiver::MergeFrom(const ::google::protobuf::Message& from) {
+ GOOGLE_CHECK_NE(&from, this);
+ const FinishedReceiver* source =
+ ::google::protobuf::internal::dynamic_cast_if_available<const FinishedReceiver*>(
+ &from);
+ if (source == NULL) {
+ ::google::protobuf::internal::ReflectionOps::Merge(from, this);
+ } else {
+ MergeFrom(*source);
+ }
+}
+
+void FinishedReceiver::MergeFrom(const FinishedReceiver& from) {
+ GOOGLE_CHECK_NE(&from, this);
+ if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) {
+ if (from.has_receiver()) {
+ mutable_receiver()->::exec::bit::FragmentHandle::MergeFrom(from.receiver());
+ }
+ if (from.has_sender()) {
+ mutable_sender()->::exec::bit::FragmentHandle::MergeFrom(from.sender());
+ }
+ }
+ mutable_unknown_fields()->MergeFrom(from.unknown_fields());
+}
+
+void FinishedReceiver::CopyFrom(const ::google::protobuf::Message& from) {
+ if (&from == this) return;
+ Clear();
+ MergeFrom(from);
+}
+
+void FinishedReceiver::CopyFrom(const FinishedReceiver& from) {
+ if (&from == this) return;
+ Clear();
+ MergeFrom(from);
+}
+
+bool FinishedReceiver::IsInitialized() const {
+
+ return true;
+}
+
+void FinishedReceiver::Swap(FinishedReceiver* other) {
+ if (other != this) {
+ std::swap(receiver_, other->receiver_);
+ std::swap(sender_, other->sender_);
+ std::swap(_has_bits_[0], other->_has_bits_[0]);
+ _unknown_fields_.Swap(&other->_unknown_fields_);
+ std::swap(_cached_size_, other->_cached_size_);
+ }
+}
+
+::google::protobuf::Metadata FinishedReceiver::GetMetadata() const {
+ protobuf_AssignDescriptorsOnce();
+ ::google::protobuf::Metadata metadata;
+ metadata.descriptor = FinishedReceiver_descriptor_;
+ metadata.reflection = FinishedReceiver_reflection_;
+ return metadata;
+}
+
+
// @@protoc_insertion_point(namespace_scope)
} // namespace control
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/protobuf/BitControl.pb.h
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/protobuf/BitControl.pb.h b/contrib/native/client/src/protobuf/BitControl.pb.h
index 06af929..011f8a8 100644
--- a/contrib/native/client/src/protobuf/BitControl.pb.h
+++ b/contrib/native/client/src/protobuf/BitControl.pb.h
@@ -44,6 +44,7 @@ class BitStatus;
class FragmentStatus;
class PlanFragment;
class WorkQueueStatus;
+class FinishedReceiver;
enum RpcType {
HANDSHAKE = 0,
@@ -51,13 +52,14 @@ enum RpcType {
GOODBYE = 2,
REQ_INIATILIZE_FRAGMENT = 3,
REQ_CANCEL_FRAGMENT = 6,
- REQ_FRAGMENT_STATUS = 7,
- REQ_BIT_STATUS = 8,
- REQ_QUERY_STATUS = 9,
- RESP_FRAGMENT_HANDLE = 10,
- RESP_FRAGMENT_STATUS = 11,
- RESP_BIT_STATUS = 12,
- RESP_QUERY_STATUS = 13
+ REQ_RECEIVER_FINISHED = 7,
+ REQ_FRAGMENT_STATUS = 8,
+ REQ_BIT_STATUS = 9,
+ REQ_QUERY_STATUS = 10,
+ RESP_FRAGMENT_HANDLE = 11,
+ RESP_FRAGMENT_STATUS = 12,
+ RESP_BIT_STATUS = 13,
+ RESP_QUERY_STATUS = 14
};
bool RpcType_IsValid(int value);
const RpcType RpcType_MIN = HANDSHAKE;
@@ -582,9 +584,9 @@ class PlanFragment : public ::google::protobuf::Message {
::exec::DrillbitEndpoint* assignment_;
::exec::DrillbitEndpoint* foreman_;
::google::protobuf::int64 mem_initial_;
- ::google::protobuf::int64 mem_max_;
bool leaf_fragment_;
::google::protobuf::int32 time_zone_;
+ ::google::protobuf::int64 mem_max_;
::google::protobuf::int64 query_start_time_;
::exec::shared::UserCredentials* credentials_;
::std::string* options_json_;
@@ -703,6 +705,102 @@ class WorkQueueStatus : public ::google::protobuf::Message {
void InitAsDefaultInstance();
static WorkQueueStatus* default_instance_;
};
+// -------------------------------------------------------------------
+
+class FinishedReceiver : public ::google::protobuf::Message {
+ public:
+ FinishedReceiver();
+ virtual ~FinishedReceiver();
+
+ FinishedReceiver(const FinishedReceiver& from);
+
+ inline FinishedReceiver& operator=(const FinishedReceiver& from) {
+ CopyFrom(from);
+ return *this;
+ }
+
+ inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const {
+ return _unknown_fields_;
+ }
+
+ inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() {
+ return &_unknown_fields_;
+ }
+
+ static const ::google::protobuf::Descriptor* descriptor();
+ static const FinishedReceiver& default_instance();
+
+ void Swap(FinishedReceiver* other);
+
+ // implements Message ----------------------------------------------
+
+ FinishedReceiver* New() const;
+ void CopyFrom(const ::google::protobuf::Message& from);
+ void MergeFrom(const ::google::protobuf::Message& from);
+ void CopyFrom(const FinishedReceiver& from);
+ void MergeFrom(const FinishedReceiver& from);
+ void Clear();
+ bool IsInitialized() const;
+
+ int ByteSize() const;
+ bool MergePartialFromCodedStream(
+ ::google::protobuf::io::CodedInputStream* input);
+ void SerializeWithCachedSizes(
+ ::google::protobuf::io::CodedOutputStream* output) const;
+ ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const;
+ int GetCachedSize() const { return _cached_size_; }
+ private:
+ void SharedCtor();
+ void SharedDtor();
+ void SetCachedSize(int size) const;
+ public:
+
+ ::google::protobuf::Metadata GetMetadata() const;
+
+ // nested types ----------------------------------------------------
+
+ // accessors -------------------------------------------------------
+
+ // optional .exec.bit.FragmentHandle receiver = 1;
+ inline bool has_receiver() const;
+ inline void clear_receiver();
+ static const int kReceiverFieldNumber = 1;
+ inline const ::exec::bit::FragmentHandle& receiver() const;
+ inline ::exec::bit::FragmentHandle* mutable_receiver();
+ inline ::exec::bit::FragmentHandle* release_receiver();
+ inline void set_allocated_receiver(::exec::bit::FragmentHandle* receiver);
+
+ // optional .exec.bit.FragmentHandle sender = 2;
+ inline bool has_sender() const;
+ inline void clear_sender();
+ static const int kSenderFieldNumber = 2;
+ inline const ::exec::bit::FragmentHandle& sender() const;
+ inline ::exec::bit::FragmentHandle* mutable_sender();
+ inline ::exec::bit::FragmentHandle* release_sender();
+ inline void set_allocated_sender(::exec::bit::FragmentHandle* sender);
+
+ // @@protoc_insertion_point(class_scope:exec.bit.control.FinishedReceiver)
+ private:
+ inline void set_has_receiver();
+ inline void clear_has_receiver();
+ inline void set_has_sender();
+ inline void clear_has_sender();
+
+ ::google::protobuf::UnknownFieldSet _unknown_fields_;
+
+ ::exec::bit::FragmentHandle* receiver_;
+ ::exec::bit::FragmentHandle* sender_;
+
+ mutable int _cached_size_;
+ ::google::protobuf::uint32 _has_bits_[(2 + 31) / 32];
+
+ friend void protobuf_AddDesc_BitControl_2eproto();
+ friend void protobuf_AssignDesc_BitControl_2eproto();
+ friend void protobuf_ShutdownFile_BitControl_2eproto();
+
+ void InitAsDefaultInstance();
+ static FinishedReceiver* default_instance_;
+};
// ===================================================================
@@ -1482,6 +1580,86 @@ inline void WorkQueueStatus::set_report_time(::google::protobuf::int64 value) {
report_time_ = value;
}
+// -------------------------------------------------------------------
+
+// FinishedReceiver
+
+// optional .exec.bit.FragmentHandle receiver = 1;
+inline bool FinishedReceiver::has_receiver() const {
+ return (_has_bits_[0] & 0x00000001u) != 0;
+}
+inline void FinishedReceiver::set_has_receiver() {
+ _has_bits_[0] |= 0x00000001u;
+}
+inline void FinishedReceiver::clear_has_receiver() {
+ _has_bits_[0] &= ~0x00000001u;
+}
+inline void FinishedReceiver::clear_receiver() {
+ if (receiver_ != NULL) receiver_->::exec::bit::FragmentHandle::Clear();
+ clear_has_receiver();
+}
+inline const ::exec::bit::FragmentHandle& FinishedReceiver::receiver() const {
+ return receiver_ != NULL ? *receiver_ : *default_instance_->receiver_;
+}
+inline ::exec::bit::FragmentHandle* FinishedReceiver::mutable_receiver() {
+ set_has_receiver();
+ if (receiver_ == NULL) receiver_ = new ::exec::bit::FragmentHandle;
+ return receiver_;
+}
+inline ::exec::bit::FragmentHandle* FinishedReceiver::release_receiver() {
+ clear_has_receiver();
+ ::exec::bit::FragmentHandle* temp = receiver_;
+ receiver_ = NULL;
+ return temp;
+}
+inline void FinishedReceiver::set_allocated_receiver(::exec::bit::FragmentHandle* receiver) {
+ delete receiver_;
+ receiver_ = receiver;
+ if (receiver) {
+ set_has_receiver();
+ } else {
+ clear_has_receiver();
+ }
+}
+
+// optional .exec.bit.FragmentHandle sender = 2;
+inline bool FinishedReceiver::has_sender() const {
+ return (_has_bits_[0] & 0x00000002u) != 0;
+}
+inline void FinishedReceiver::set_has_sender() {
+ _has_bits_[0] |= 0x00000002u;
+}
+inline void FinishedReceiver::clear_has_sender() {
+ _has_bits_[0] &= ~0x00000002u;
+}
+inline void FinishedReceiver::clear_sender() {
+ if (sender_ != NULL) sender_->::exec::bit::FragmentHandle::Clear();
+ clear_has_sender();
+}
+inline const ::exec::bit::FragmentHandle& FinishedReceiver::sender() const {
+ return sender_ != NULL ? *sender_ : *default_instance_->sender_;
+}
+inline ::exec::bit::FragmentHandle* FinishedReceiver::mutable_sender() {
+ set_has_sender();
+ if (sender_ == NULL) sender_ = new ::exec::bit::FragmentHandle;
+ return sender_;
+}
+inline ::exec::bit::FragmentHandle* FinishedReceiver::release_sender() {
+ clear_has_sender();
+ ::exec::bit::FragmentHandle* temp = sender_;
+ sender_ = NULL;
+ return temp;
+}
+inline void FinishedReceiver::set_allocated_sender(::exec::bit::FragmentHandle* sender) {
+ delete sender_;
+ sender_ = sender;
+ if (sender) {
+ set_has_sender();
+ } else {
+ clear_has_sender();
+ }
+}
+
// @@protoc_insertion_point(namespace_scope)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/protobuf/Types.pb.cc
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/protobuf/Types.pb.cc b/contrib/native/client/src/protobuf/Types.pb.cc
index c7fa2ea..a14f404 100644
--- a/contrib/native/client/src/protobuf/Types.pb.cc
+++ b/contrib/native/client/src/protobuf/Types.pb.cc
@@ -91,7 +91,7 @@ void protobuf_AddDesc_Types_2eproto() {
"inor_type\030\001 \001(\0162\021.common.MinorType\022\036\n\004mo"
"de\030\002 \001(\0162\020.common.DataMode\022\r\n\005width\030\003 \001("
"\005\022\021\n\tprecision\030\004 \001(\005\022\r\n\005scale\030\005 \001(\005\022\020\n\010t"
- "imeZone\030\006 \001(\005*\374\003\n\tMinorType\022\010\n\004LATE\020\000\022\007\n"
+ "imeZone\030\006 \001(\005*\220\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n"
"\003MAP\020\001\022\013\n\007TINYINT\020\003\022\014\n\010SMALLINT\020\004\022\007\n\003INT"
"\020\005\022\n\n\006BIGINT\020\006\022\014\n\010DECIMAL9\020\007\022\r\n\tDECIMAL1"
"8\020\010\022\023\n\017DECIMAL28SPARSE\020\t\022\023\n\017DECIMAL38SPA"
@@ -104,9 +104,10 @@ void protobuf_AddDesc_Types_2eproto() {
"\036\022\t\n\005UINT4\020\037\022\t\n\005UINT8\020 \022\022\n\016DECIMAL28DENS"
"E\020!\022\022\n\016DECIMAL38DENSE\020\"\022\016\n\nDM_UNKNOWN\020%\022"
"\020\n\014INTERVALYEAR\020&\022\017\n\013INTERVALDAY\020\'\022\010\n\004LI"
- "ST\020(*=\n\010DataMode\022\017\n\013DM_OPTIONAL\020\000\022\017\n\013DM_"
- "REQUIRED\020\001\022\017\n\013DM_REPEATED\020\002B-\n\035org.apach"
- "e.drill.common.typesB\nTypeProtosH\001", 794);
+ "ST\020(\022\022\n\016GENERIC_OBJECT\020)*=\n\010DataMode\022\017\n\013"
+ "DM_OPTIONAL\020\000\022\017\n\013DM_REQUIRED\020\001\022\017\n\013DM_REP"
+ "EATED\020\002B-\n\035org.apache.drill.common.types"
+ "B\nTypeProtosH\001", 814);
::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
"Types.proto", &protobuf_RegisterTypes);
MajorType::default_instance_ = new MajorType();
@@ -162,6 +163,7 @@ bool MinorType_IsValid(int value) {
case 38:
case 39:
case 40:
+ case 41:
return true;
default:
return false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/protobuf/UserBitShared.pb.cc
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
index 88507e5..0e7649f 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
@@ -532,24 +532,25 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
"\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmen"
"tState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATI"
"ON\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCE"
- "LLED\020\004\022\n\n\006FAILED\020\005*\372\004\n\020CoreOperatorType\022"
+ "LLED\020\004\022\n\n\006FAILED\020\005*\224\005\n\020CoreOperatorType\022"
"\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001"
"\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH"
"_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITIO"
"N_SENDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVE"
"R\020\010\022\034\n\030ORDERED_PARTITION_SENDER\020\t\022\013\n\007PRO"
- "JECT\020\n\022\023\n\017RANDOM_RECEIVER\020\013\022\020\n\014RANGE_SEN"
- "DER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_RE"
- "MOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_"
- "N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t"
- "\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_G"
- "ROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM"
- "_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQ"
- "UET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEX"
- "T_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SU"
- "B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017CO"
- "MPLEX_TO_JSON\020\037B.\n\033org.apache.drill.exec"
- ".protoB\rUserBitSharedH\001", 3543);
+ "JECT\020\n\022\026\n\022UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_"
+ "SENDER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR"
+ "_REMOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nT"
+ "OP_N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020"
+ "\022\022\t\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_RO"
+ "W_GROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYS"
+ "TEM_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016P"
+ "ARQUET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013"
+ "TEXT_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON"
+ "_SUB_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n"
+ "\017COMPLEX_TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020"
+ " B.\n\033org.apache.drill.exec.protoB\rUserBi"
+ "tSharedH\001", 3569);
::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
"UserBitShared.proto", &protobuf_RegisterTypes);
UserCredentials::default_instance_ = new UserCredentials();
@@ -677,6 +678,7 @@ bool CoreOperatorType_IsValid(int value) {
case 29:
case 30:
case 31:
+ case 32:
return true;
default:
return false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/contrib/native/client/src/protobuf/UserBitShared.pb.h
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h
index 6ef17c3..e7d40b1 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.h
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h
@@ -171,7 +171,7 @@ enum CoreOperatorType {
MERGING_RECEIVER = 8,
ORDERED_PARTITION_SENDER = 9,
PROJECT = 10,
- RANDOM_RECEIVER = 11,
+ UNORDERED_RECEIVER = 11,
RANGE_SENDER = 12,
SCREEN = 13,
SELECTION_VECTOR_REMOVER = 14,
@@ -191,11 +191,12 @@ enum CoreOperatorType {
TEXT_SUB_SCAN = 28,
JSON_SUB_SCAN = 29,
INFO_SCHEMA_SUB_SCAN = 30,
- COMPLEX_TO_JSON = 31
+ COMPLEX_TO_JSON = 31,
+ PRODUCER_CONSUMER = 32
};
bool CoreOperatorType_IsValid(int value);
const CoreOperatorType CoreOperatorType_MIN = SINGLE_SENDER;
-const CoreOperatorType CoreOperatorType_MAX = COMPLEX_TO_JSON;
+const CoreOperatorType CoreOperatorType_MAX = PRODUCER_CONSUMER;
const int CoreOperatorType_ARRAYSIZE = CoreOperatorType_MAX + 1;
const ::google::protobuf::EnumDescriptor* CoreOperatorType_descriptor();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc25504d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b46fde8..deb95fe 100644
--- a/pom.xml
+++ b/pom.xml
@@ -185,6 +185,7 @@
<exclude>**/*.pb.cc</exclude>
<exclude>**/*.pb.h</exclude>
<exclude>**/*.linux</exclude>
+ <exclude>**/client/build/**</exclude>
</excludes>
</configuration>
</plugin>
[46/47] git commit: DRILL-1354: Fix date truncate functions to use
toString helper functions. Minor cleanup: move the truncate logic in a
separate template
Posted by ja...@apache.org.
DRILL-1354: Fix date truncate functions to use toString helper functions.
Minor cleanup: move the truncate logic in a separate template
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/46de8816
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/46de8816
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/46de8816
Branch: refs/heads/master
Commit: 46de8816f4b197d54d937d0a190bae293c17d1ac
Parents: d8a770f
Author: Mehant Baid <me...@gmail.com>
Authored: Wed Aug 27 16:47:54 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 18:44:09 2014 -0700
----------------------------------------------------------------------
.../DateDateArithmeticFunctions.java | 40 ----------
.../DateTruncFunctions.java | 84 ++++++++++++++++++++
.../main/codegen/templates/ValueHolders.java | 6 +-
.../drill/jdbc/test/TestFunctionsQuery.java | 1 -
4 files changed, 89 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/46de8816/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
index e119958..6c03f3b 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateDateArithmeticFunctions.java
@@ -75,45 +75,5 @@ public static class G${type}Difference implements DrillSimpleFunc {
</#if>
}
}
-
-@SuppressWarnings("unused")
-@FunctionTemplate(names = "date_trunc", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
-public static class G${type}DateTrunc implements DrillSimpleFunc {
-
- @Param VarCharHolder left;
- @Param ${type}Holder right;
- @Output ${type}Holder out;
- @Workspace org.joda.time.MutableDateTime dateTime;
-
- public void setup(RecordBatch incoming) {
- dateTime = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.UTC);
- }
-
- public void eval() {
- dateTime.setMillis(right.value);
- <#if type != "Time">
- if (left.toString().equalsIgnoreCase("YEAR")) dateTime.setRounding(dateTime.getChronology().year());
- else if (left.toString().equalsIgnoreCase("MONTH")) dateTime.setRounding(dateTime.getChronology().monthOfYear());
- else if (left.toString().equalsIgnoreCase("DAY")) dateTime.setRounding(dateTime.getChronology().dayOfMonth());
- else
- </#if>
- <#if type != "Date">
- if (left.toString().equalsIgnoreCase("HOUR")) dateTime.setRounding(dateTime.getChronology().hourOfDay());
- else if (left.toString().equalsIgnoreCase("MINUTE")) dateTime.setRounding(dateTime.getChronology().minuteOfHour());
- else if (left.toString().equalsIgnoreCase("SECOND")) dateTime.setRounding(dateTime.getChronology().secondOfMinute());
- else
- </#if>
- <#if type == "TimeStamp" || type == "TimeStampTZ">
- throw new UnsupportedOperationException("date_trunc function supports the following time units for TimeStamp(TZ): YEAR, MONTH, DAY, HOUR, MINUTE, SECOND");
- out.value = dateTime.getMillis();
- <#elseif type == "Date">
- throw new UnsupportedOperationException("date_trunc function supports the following time units for Date: YEAR, MONTH, DAY");
- out.value = dateTime.getMillis();
- <#elseif type == "Time">
- throw new UnsupportedOperationException("date_trunc function supports the following time units for Time: HOUR, MINUTE, SECOND");
- out.value = (int) dateTime.getMillis();
- </#if>
- }
-}
}
</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/46de8816/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateTruncFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateTruncFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateTruncFunctions.java
new file mode 100644
index 0000000..b9cd0c8
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateTruncFunctions.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.exec.expr.annotations.Workspace;
+
+<@pp.dropOutputFile />
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/GDateTimeTruncateFunctions.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+
+import io.netty.buffer.ByteBuf;
+
+public class GDateTimeTruncateFunctions {
+
+<#list dateIntervalFunc.dates as type>
+
+@SuppressWarnings("unused")
+@FunctionTemplate(names = "date_trunc", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+public static class G${type}DateTrunc implements DrillSimpleFunc {
+
+ @Param VarCharHolder left;
+ @Param ${type}Holder right;
+ @Output ${type}Holder out;
+ @Workspace org.joda.time.MutableDateTime dateTime;
+
+ public void setup(RecordBatch incoming) {
+ dateTime = new org.joda.time.MutableDateTime(org.joda.time.DateTimeZone.UTC);
+ }
+
+ public void eval() {
+ dateTime.setMillis(right.value);
+ <#if type != "Time">
+ if (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(left.start, left.end, left.buffer).equalsIgnoreCase("YEAR")) dateTime.setRounding(dateTime.getChronology().year());
+ else if (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(left.start, left.end, left.buffer).equalsIgnoreCase("MONTH")) dateTime.setRounding(dateTime.getChronology().monthOfYear());
+ else if (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(left.start, left.end, left.buffer).equalsIgnoreCase("DAY")) dateTime.setRounding(dateTime.getChronology().dayOfMonth());
+ else
+ </#if>
+ <#if type != "Date">
+ if (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(left.start, left.end, left.buffer).equalsIgnoreCase("HOUR")) dateTime.setRounding(dateTime.getChronology().hourOfDay());
+ else if (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(left.start, left.end, left.buffer).equalsIgnoreCase("MINUTE")) dateTime.setRounding(dateTime.getChronology().minuteOfHour());
+ else if (org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(left.start, left.end, left.buffer).equalsIgnoreCase("SECOND")) dateTime.setRounding(dateTime.getChronology().secondOfMinute());
+ else
+ </#if>
+ <#if type == "TimeStamp" || type == "TimeStampTZ">
+ throw new UnsupportedOperationException("date_trunc function supports the following time units for TimeStamp(TZ): YEAR, MONTH, DAY, HOUR, MINUTE, SECOND");
+ out.value = dateTime.getMillis();
+ <#elseif type == "Date">
+ throw new UnsupportedOperationException("date_trunc function supports the following time units for Date: YEAR, MONTH, DAY");
+ out.value = dateTime.getMillis();
+ <#elseif type == "Time">
+ throw new UnsupportedOperationException("date_trunc function supports the following time units for Time: HOUR, MINUTE, SECOND");
+ out.value = (int) dateTime.getMillis();
+ </#if>
+ }
+}
+</#list>
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/46de8816/exec/java-exec/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ValueHolders.java b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
index 1902843..4151bbe 100644
--- a/exec/java-exec/src/main/codegen/templates/ValueHolders.java
+++ b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
@@ -95,7 +95,11 @@ public final class ${className} implements ValueHolder{
public int hashCode(){
throw new UnsupportedOperationException();
}
-
+
+ /*
+ * Reason for deprecation is that ValueHolders are potential scalar replacements
+ * and hence we don't want any methods to be invoked on them.
+ */
@Deprecated
public String toString(){
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/46de8816/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index 0e40d11..5aee392 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -539,7 +539,6 @@ public class TestFunctionsQuery {
}
@Test
- @Ignore ("toString on varchar is not supported. date trunc functions seem to be using that")
public void testDateTrunc() throws Exception {
String query = "select "
+ "date_trunc('MINUTE', time '2:30:21.5') as TIME1, "
[05/47] git commit: Use Direct Memory in Parquet Writer
Posted by ja...@apache.org.
Use Direct Memory in Parquet Writer
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/fab96c34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/fab96c34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/fab96c34
Branch: refs/heads/master
Commit: fab96c34cfc392b60fffc75b09c5d1927a72f33a
Parents: a0a8256
Author: Parth Chandra <pc...@maprtech.com>
Authored: Wed Aug 20 13:37:27 2014 -0700
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Wed Aug 20 14:16:15 2014 -0700
----------------------------------------------------------------------
exec/java-exec/pom.xml | 12 ++--
.../ParquetDirectByteBufferAllocator.java | 73 ++++++++++++++++++++
.../exec/store/parquet/ParquetFormatPlugin.java | 8 ++-
.../exec/store/parquet/ParquetRecordWriter.java | 47 +++++++++----
.../ColumnChunkPageWriteStoreExposer.java | 16 ++++-
.../exec/store/parquet/TestFileGenerator.java | 11 ++-
6 files changed, 142 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fab96c34/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 685f2fe..903f953 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -125,7 +125,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-column</artifactId>
- <version>1.5.1-drill-r2</version>
+ <version>1.5.1-drill-r3</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -140,7 +140,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
- <version>1.5.1-drill-r2</version>
+ <version>1.5.1-drill-r3</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -170,7 +170,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-common</artifactId>
- <version>1.5.1-drill-r2</version>
+ <version>1.5.1-drill-r3</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -185,7 +185,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-jackson</artifactId>
- <version>1.5.1-drill-r2</version>
+ <version>1.5.1-drill-r3</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -200,7 +200,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-encoding</artifactId>
- <version>1.5.1-drill-r2</version>
+ <version>1.5.1-drill-r3</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -215,7 +215,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-generator</artifactId>
- <version>1.5.1-drill-r2</version>
+ <version>1.5.1-drill-r3</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fab96c34/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java
new file mode 100644
index 0000000..1a49dcd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetDirectByteBufferAllocator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.store.parquet;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.drill.exec.ops.OperatorContext;
+import parquet.bytes.ByteBufferAllocator;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class ParquetDirectByteBufferAllocator implements ByteBufferAllocator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetDirectByteBufferAllocator.class);
+
+ private OperatorContext oContext;
+ private HashMap<Integer, ByteBuf> allocatedBuffers = new HashMap<Integer, ByteBuf>();
+
+ public ParquetDirectByteBufferAllocator(OperatorContext o){
+ oContext=o;
+ }
+
+
+ @Override
+ public ByteBuffer allocate(int sz) {
+ ByteBuf bb = oContext.getAllocator().buffer(sz);
+ ByteBuffer b = bb.nioBuffer(0, sz);
+ allocatedBuffers.put(System.identityHashCode(b), bb);
+ logger.debug("ParquetDirectByteBufferAllocator: Allocated "+sz+" bytes. Allocated ByteBuffer id: "+System.identityHashCode(b));
+ return b;
+ }
+
+ @Override
+ public void release(ByteBuffer b) {
+ Integer id = System.identityHashCode(b);
+ ByteBuf bb = allocatedBuffers.get(id);
+ // The ByteBuffer passed in may already have been freed or not allocated by this allocator.
+ // If it is not found in the allocated buffers, do nothing
+ if(bb!=null) {
+ logger.debug("ParquetDirectByteBufferAllocator: Freed byte buffer. Allocated ByteBuffer id: "+System.identityHashCode(b));
+ bb.release();
+ allocatedBuffers.remove(id);
+ }
+ }
+
+ public void clear(){
+ Iterator it = allocatedBuffers.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ Integer id = (Integer)pair.getKey();
+ ByteBuf bb = allocatedBuffers.get(id);
+ bb.release();
+ it.remove();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fab96c34/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index 37d6403..eb07d79 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -27,7 +27,9 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.impl.WriterRecordBatch;
@@ -71,7 +73,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
private final ParquetFormatConfig config;
private final StoragePluginConfig storageConfig;
private final String name;
-
+
public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig){
this(name, context, fs, storageConfig, new ParquetFormatConfig());
}
@@ -118,7 +120,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
return new ParquetWriter(child, location, this);
}
- public RecordWriter getRecordWriter(FragmentContext context, ParquetWriter writer) throws IOException {
+ public RecordWriter getRecordWriter(FragmentContext context, ParquetWriter writer) throws IOException, OutOfMemoryException {
Map<String, String> options = Maps.newHashMap();
options.put("location", writer.getLocation());
@@ -131,7 +133,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
options.put(ExecConstants.PARQUET_BLOCK_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_BLOCK_SIZE).num_val.toString());
- RecordWriter recordWriter = new ParquetRecordWriter();
+ RecordWriter recordWriter = new ParquetRecordWriter(context, writer);
recordWriter.init(options);
return recordWriter;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fab96c34/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index 94ccc13..2c5f232 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -21,14 +21,14 @@ import com.google.common.collect.Lists;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.expr.holders.ComplexHolder;
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.EventBasedRecordWriter;
import org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter;
import org.apache.drill.exec.store.ParquetOutputRecordWriter;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -67,7 +67,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
private static final int MINIMUM_RECORD_COUNT_FOR_CHECK = 100;
private static final int MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
- private ParquetFileWriter w;
+ private ParquetFileWriter parquetFileWriter;
private MessageType schema;
private Map<String, String> extraMetaData = new HashMap();
private int blockSize;
@@ -91,6 +91,13 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
private String location;
private String prefix;
private int index = 0;
+ private OperatorContext oContext;
+ private ParquetDirectByteBufferAllocator allocator;
+
+ public ParquetRecordWriter(FragmentContext context, ParquetWriter writer) throws OutOfMemoryException{
+ super();
+ this.oContext=new OperatorContext(writer, context);
+ }
@Override
public void init(Map<String, String> writerOptions) throws IOException {
@@ -121,11 +128,15 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
schema = new MessageType("root", types);
Path fileName = new Path(location, prefix + "_" + index + ".parquet");
- w = new ParquetFileWriter(conf, schema, fileName);
- w.start();
+ parquetFileWriter = new ParquetFileWriter(conf, schema, fileName);
+ parquetFileWriter.start();
int initialBlockBufferSize = max(MINIMUM_BUFFER_SIZE, blockSize / this.schema.getColumns().size() / 5);
- pageStore = ColumnChunkPageWriteStoreExposer.newColumnChunkPageWriteStore(codec, pageSize, this.schema, initialBlockBufferSize);
+ pageStore = ColumnChunkPageWriteStoreExposer.newColumnChunkPageWriteStore(this.oContext,
+ codec,
+ pageSize,
+ this.schema,
+ initialBlockBufferSize);
int initialPageBufferSize = max(MINIMUM_BUFFER_SIZE, min(pageSize + pageSize / 10, initialBlockBufferSize));
store = new ColumnWriteStoreImpl(pageStore, pageSize, initialPageBufferSize, dictionaryPageSize, enableDictionary, writerVersion);
MessageColumnIO columnIO = new ColumnIOFactory(validating).getColumnIO(this.schema);
@@ -162,12 +173,14 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
}
private void flush() throws IOException {
- w.startBlock(recordCount);
+ parquetFileWriter.startBlock(recordCount);
store.flush();
- ColumnChunkPageWriteStoreExposer.flushPageStore(pageStore, w);
+ ColumnChunkPageWriteStoreExposer.flushPageStore(pageStore, parquetFileWriter);
recordCount = 0;
- w.endBlock();
- w.end(extraMetaData);
+ parquetFileWriter.endBlock();
+ parquetFileWriter.end(extraMetaData);
+ store.close();
+ ColumnChunkPageWriteStoreExposer.close(pageStore);
store = null;
pageStore = null;
index++;
@@ -274,7 +287,17 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
@Override
public void cleanup() throws IOException {
if (recordCount > 0) {
- flush();
+ parquetFileWriter.startBlock(recordCount);
+ store.flush();
+ ColumnChunkPageWriteStoreExposer.flushPageStore(pageStore, parquetFileWriter);
+ recordCount = 0;
+ parquetFileWriter.endBlock();
+ parquetFileWriter.end(extraMetaData);
+ }
+ store.close();
+ ColumnChunkPageWriteStoreExposer.close(pageStore);
+ if(oContext!=null){
+ oContext.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fab96c34/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkPageWriteStoreExposer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkPageWriteStoreExposer.java b/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkPageWriteStoreExposer.java
index 54f647a..0ffc3ba 100644
--- a/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkPageWriteStoreExposer.java
+++ b/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkPageWriteStoreExposer.java
@@ -17,6 +17,8 @@
*/
package parquet.hadoop;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.store.parquet.ParquetDirectByteBufferAllocator;
import org.apache.hadoop.conf.Configuration;
import parquet.column.page.PageWriteStore;
import parquet.hadoop.CodecFactory.BytesCompressor;
@@ -27,13 +29,23 @@ import java.io.IOException;
public class ColumnChunkPageWriteStoreExposer {
- public static ColumnChunkPageWriteStore newColumnChunkPageWriteStore(CompressionCodecName codec, int pageSize, MessageType schema, int initialSize) {
+ public static ColumnChunkPageWriteStore newColumnChunkPageWriteStore(OperatorContext oContext,
+ CompressionCodecName codec,
+ int pageSize,
+ MessageType schema,
+ int initialSize) {
BytesCompressor compressor = new CodecFactory(new Configuration()).getCompressor(codec, pageSize);
- return new ColumnChunkPageWriteStore(compressor, schema, initialSize);
+ return new ColumnChunkPageWriteStore(compressor, schema, initialSize, new ParquetDirectByteBufferAllocator(oContext));
}
public static void flushPageStore(PageWriteStore pageStore, ParquetFileWriter w) throws IOException {
((ColumnChunkPageWriteStore) pageStore).flushToFileWriter(w);
}
+ public static void close(PageWriteStore pageStore) throws IOException {
+ ((ColumnChunkPageWriteStore) pageStore).close();
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fab96c34/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
index 3c0287d..fb1ea60 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import parquet.bytes.BytesInput;
+import parquet.bytes.DirectByteBufferAllocator;
import parquet.column.ColumnDescriptor;
import parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter;
import parquet.hadoop.ParquetFileWriter;
@@ -183,8 +184,14 @@ public class TestFileGenerator {
w.startColumn(c1, props.recordsPerRowGroup, codec);
int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages);
byte[] bytes;
- RunLengthBitPackingHybridValuesWriter defLevels = new RunLengthBitPackingHybridValuesWriter(MAX_EXPECTED_BIT_WIDTH_FOR_DEFINITION_LEVELS, valsPerPage);
- RunLengthBitPackingHybridValuesWriter repLevels = new RunLengthBitPackingHybridValuesWriter(MAX_EXPECTED_BIT_WIDTH_FOR_DEFINITION_LEVELS, valsPerPage);
+ RunLengthBitPackingHybridValuesWriter defLevels = new RunLengthBitPackingHybridValuesWriter(
+ MAX_EXPECTED_BIT_WIDTH_FOR_DEFINITION_LEVELS,
+ valsPerPage,
+ new DirectByteBufferAllocator());
+ RunLengthBitPackingHybridValuesWriter repLevels = new RunLengthBitPackingHybridValuesWriter(
+ MAX_EXPECTED_BIT_WIDTH_FOR_DEFINITION_LEVELS,
+ valsPerPage,
+ new DirectByteBufferAllocator());
// for variable length binary fields
int bytesNeededToEncodeLength = 4;
if ((int) fieldInfo.bitLength > 0) {
[39/47] git commit: enable mapr profile hbase
Posted by ja...@apache.org.
enable mapr profile hbase
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ceb58c41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ceb58c41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ceb58c41
Branch: refs/heads/master
Commit: ceb58c4149f00de4729ff7350b576823e71fbdfc
Parents: 496b5d0
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Aug 27 17:53:55 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 18:30:57 2014 -0700
----------------------------------------------------------------------
distribution/pom.xml | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ceb58c41/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 069f3ac..a63dc6c 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -132,6 +132,10 @@
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ <exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
@@ -265,7 +269,6 @@
<groupId>com.mapr.hadoop</groupId>
<artifactId>maprfs</artifactId>
</dependency>
- <!--
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
@@ -284,6 +287,11 @@
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+
+ <exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
@@ -330,6 +338,10 @@
<artifactId>mapr-hbase</artifactId>
<exclusions>
<exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
@@ -339,7 +351,7 @@
</exclusion>
</exclusions>
</dependency>
- -->
+
</dependencies>
<build>
<plugins>
[11/47] git commit: DRILL-1322: Fix memory leak when all records in a
batch are filtered
Posted by ja...@apache.org.
DRILL-1322: Fix memory leak when all records in a batch are filtered
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2275da9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2275da9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2275da9c
Branch: refs/heads/master
Commit: 2275da9c2c0aaf072172bff302f8b11800989e35
Parents: f1136ab
Author: Mehant Baid <me...@gmail.com>
Authored: Fri Aug 22 01:03:47 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 12:03:26 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/record/WritableBatch.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2275da9c/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
index 97f665a..b55ed82 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
@@ -132,8 +132,10 @@ public class WritableBatch {
metadata.add(vv.getMetadata());
// don't try to get the buffers if we don't have any records. It is possible the buffers are dead buffers.
- if (recordCount == 0)
+ if (recordCount == 0) {
+ vv.clear();
continue;
+ }
for (ByteBuf b : vv.getBuffers()) {
buffers.add(b);
[03/47] git commit: DRILL-1281: Use ByteBuffer read codepath in
complex parquet reader
Posted by ja...@apache.org.
DRILL-1281: Use ByteBuffer read codepath in complex parquet reader
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c3b15af0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c3b15af0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c3b15af0
Branch: refs/heads/master
Commit: c3b15af0292270e75a3f98a9225e7dceb24c96c3
Parents: 02e4824
Author: Steven Phillips <sp...@maprtech.com>
Authored: Mon Aug 11 11:58:43 2014 -0700
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Mon Aug 18 14:46:43 2014 -0700
----------------------------------------------------------------------
contrib/storage-hbase/pom.xml | 26 +++++++++-
distribution/pom.xml | 13 ++---
distribution/src/assemble/bin.xml | 2 +-
distribution/src/resources/drill-config.sh | 27 -----------
exec/java-exec/pom.xml | 50 ++++++++++++++++++++
.../exec/store/parquet2/DrillParquetReader.java | 11 ++++-
.../parquet/hadoop/ColumnChunkIncReadStore.java | 46 ++++++++++++------
pom.xml | 4 +-
8 files changed, 125 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/contrib/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml
index 82bf4bb..642bc34 100644
--- a/contrib/storage-hbase/pom.xml
+++ b/contrib/storage-hbase/pom.xml
@@ -180,6 +180,18 @@
<artifactId>asm</artifactId>
<groupId>asm</groupId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- test dependencies -->
@@ -193,6 +205,18 @@
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -232,7 +256,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
- <version>0.94.13-mapr-1401-m7-3.0.2</version>
+ <version>0.94.17-mapr-1405-m7-4.0.0-FCS</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 87738bb..1429902 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -65,6 +65,12 @@
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-storage-hbase</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.drill.contrib.storage-hive</groupId>
@@ -72,14 +78,9 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>2.4.1</version>
- </dependency>
- <dependency>
<groupId>de.huxhorn.lilith</groupId>
<artifactId>de.huxhorn.lilith.logback.appender.multiplex-classic</artifactId>
- <version>0.9.42.1</version>
+ <version>0.9.44</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index c1675e9..6d7fb6b 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -90,7 +90,7 @@
<unpack>false</unpack>
<useProjectArtifact>false</useProjectArtifact>
<includes>
- <include>com.google.protobuf:protobuf-java:jar:2.4.1</include>
+ <include>com.google.protobuf:protobuf-java:jar:2.5</include>
</includes>
<scope>test</scope>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/distribution/src/resources/drill-config.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh
index 795e97a..621f947 100644
--- a/distribution/src/resources/drill-config.sh
+++ b/distribution/src/resources/drill-config.sh
@@ -88,33 +88,6 @@ if [ ! -d $DRILL_LOG_DIR ]; then
mkdir -p $DRILL_LOG_DIR
fi
-# If HADOOP_HOME is specified, add all Hadoop jars except those
-# specifically excluded in $DRILL_HOME/bin/hadoop-excludes.txt
-if [ "${HADOOP_HOME}x" != "x" ] ; then
- HADOOP_CLASSPATH=""
- for jar in `ls $HADOOP_HOME/lib/*jar` ; do
- echo $jar | grep -v -f $DRILL_HOME/bin/hadoop-excludes.txt >/dev/null
- if [ "$?" -eq "0" ] ; then
- HADOOP_CLASSPATH=$jar:$HADOOP_CLASSPATH
- fi
- done
- HADOOP_CLASSPATH=$HADOOP_HOME/conf:$HADOOP_CLASSPATH
-fi
-if [ "${HBASE_HOME}x" != "x" ]
-then
- HBASE_CLASSPATH=""
- for jar in `ls $HBASE_HOME/*jar`
- do
- echo $jar | grep -v -f $DRILL_HOME/bin/hadoop-excludes.txt >/dev/null
- if [ "$?" -eq "0" ]
- then
- HBASE_CLASSPATH=$jar:$HBASE_CLASSPATH
- fi
- done
- HBASE_CLASSPATH=$HBASE_HOME/conf:$HBASE_CLASSPATH
- export HBASE_CLASSPATH=$HBASE_CLASSPATH
-fi
-
# Add Drill conf folder at the beginning of the classpath
CP=$DRILL_CONF_DIR
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 21dfc67..685f2fe 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -357,6 +357,10 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
@@ -367,6 +371,16 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.mapr.hadoop</groupId>
@@ -393,6 +407,42 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 0d2a225..f47acab 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -46,6 +46,7 @@ import parquet.schema.GroupType;
import parquet.schema.MessageType;
import parquet.schema.Type;
+import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -61,6 +62,7 @@ public class DrillParquetReader implements RecordReader {
private RowGroupReadEntry entry;
private List<SchemaPath> columns;
private VectorContainerWriter writer;
+ private ColumnChunkIncReadStore pageReadStore;
private parquet.io.RecordReader<Void> recordReader;
private DrillParquetRecordMaterializer recordMaterializer;
private int recordCount;
@@ -142,8 +144,8 @@ public class DrillParquetReader implements RecordReader {
recordCount = (int) blockMetaData.getRowCount();
- ColumnChunkIncReadStore pageReadStore = new ColumnChunkIncReadStore(recordCount,
- codecFactoryExposer.getCodecFactory(), fs, filePath);
+ pageReadStore = new ColumnChunkIncReadStore(recordCount,
+ codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);
for (String[] path : schema.getPaths()) {
Type type = schema.getType(path);
@@ -203,6 +205,11 @@ public class DrillParquetReader implements RecordReader {
@Override
public void cleanup() {
+ try {
+ pageReadStore.close();
+ } catch (IOException e) {
+ logger.warn("Failure while closing PageReadStore", e);
+ }
}
public void setOperatorContext(OperatorContext operatorContext) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java b/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
index e5a477b..379d3e6 100644
--- a/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
+++ b/exec/java-exec/src/main/java/parquet/hadoop/ColumnChunkIncReadStore.java
@@ -17,6 +17,8 @@
*/
package parquet.hadoop;
+import io.netty.buffer.ByteBuf;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -35,8 +37,10 @@ import parquet.format.converter.ParquetMetadataConverter;
import parquet.hadoop.CodecFactory.BytesDecompressor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.hadoop.metadata.FileMetaData;
+import parquet.hadoop.util.CompatibilityUtil;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -48,13 +52,15 @@ public class ColumnChunkIncReadStore implements PageReadStore {
private static ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
private CodecFactory codecFactory = new CodecFactory(new Configuration());
+ private BufferAllocator allocator;
private FileSystem fs;
private Path path;
private long rowCount;
private List<FSDataInputStream> streams = new ArrayList();
- public ColumnChunkIncReadStore(long rowCount, CodecFactory codecFactory, FileSystem fs, Path path) {
+ public ColumnChunkIncReadStore(long rowCount, CodecFactory codecFactory, BufferAllocator allocator, FileSystem fs, Path path) {
this.codecFactory = codecFactory;
+ this.allocator = allocator;
this.fs = fs;
this.path = path;
this.rowCount = rowCount;
@@ -64,6 +70,7 @@ public class ColumnChunkIncReadStore implements PageReadStore {
public class ColumnChunkIncPageReader implements PageReader {
ColumnChunkMetaData metaData;
+ ColumnDescriptor columnDescriptor;
long fileOffset;
long size;
private long valueReadSoFar = 0;
@@ -72,16 +79,11 @@ public class ColumnChunkIncReadStore implements PageReadStore {
private FSDataInputStream in;
private BytesDecompressor decompressor;
- public ColumnChunkIncPageReader(ColumnChunkMetaData metaData, FSDataInputStream in) {
- this.metaData = metaData;
- this.size = metaData.getTotalSize();
- this.fileOffset = metaData.getStartingPos();
- this.in = in;
- this.decompressor = codecFactory.getDecompressor(metaData.getCodec());
- }
+ private ByteBuf lastPage;
- public ColumnChunkIncPageReader(ColumnChunkMetaData metaData, FSDataInputStream in, CodecFactory codecFactory) {
+ public ColumnChunkIncPageReader(ColumnChunkMetaData metaData, ColumnDescriptor columnDescriptor, FSDataInputStream in) {
this.metaData = metaData;
+ this.columnDescriptor = columnDescriptor;
this.size = metaData.getTotalSize();
this.fileOffset = metaData.getStartingPos();
this.in = in;
@@ -104,13 +106,9 @@ public class ColumnChunkIncReadStore implements PageReadStore {
pageHeader.getDictionary_page_header().getNum_values(),
parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
);
- System.out.println(dictionaryPage);
} catch (IOException e) {
throw new RuntimeException(e);
}
-// if (dictionaryPage == null) {
-// throw new RuntimeException("Dictionary page null");
-// }
}
return dictionaryPage;
}
@@ -123,6 +121,10 @@ public class ColumnChunkIncReadStore implements PageReadStore {
@Override
public Page readPage() {
try {
+ if (lastPage != null) {
+ lastPage.release();
+ lastPage = null;
+ }
while(valueReadSoFar < metaData.getValueCount()) {
PageHeader pageHeader = Util.readPageHeader(in);
switch (pageHeader.type) {
@@ -140,10 +142,15 @@ public class ColumnChunkIncReadStore implements PageReadStore {
break;
case DATA_PAGE:
valueReadSoFar += pageHeader.data_page_header.getNum_values();
+ ByteBuf buf = allocator.buffer(pageHeader.compressed_page_size);
+ lastPage = buf;
+ ByteBuffer buffer = buf.nioBuffer(0, pageHeader.compressed_page_size);
+ CompatibilityUtil.getBuf(in, buffer, pageHeader.compressed_page_size);
return new Page(
- decompressor.decompress(BytesInput.from(in,pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
+ decompressor.decompress(BytesInput.from(buffer, 0, pageHeader.compressed_page_size), pageHeader.compressed_page_size),
pageHeader.data_page_header.num_values,
pageHeader.uncompressed_page_size,
+ parquetMetadataConverter.fromParquetStatistics(pageHeader.data_page_header.statistics, columnDescriptor.getType()),
parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
parquetMetadataConverter.getEncoding(pageHeader.data_page_header.encoding)
@@ -159,6 +166,12 @@ public class ColumnChunkIncReadStore implements PageReadStore {
throw new RuntimeException(e);
}
}
+
+ void close() {
+ if (lastPage != null) {
+ lastPage.release();
+ }
+ }
}
private Map<ColumnDescriptor, ColumnChunkIncPageReader> columns = new HashMap();
@@ -167,7 +180,7 @@ public class ColumnChunkIncReadStore implements PageReadStore {
FSDataInputStream in = fs.open(path);
streams.add(in);
in.seek(metaData.getStartingPos());
- ColumnChunkIncPageReader reader = new ColumnChunkIncPageReader(metaData, in);
+ ColumnChunkIncPageReader reader = new ColumnChunkIncPageReader(metaData, descriptor, in);
columns.put(descriptor, reader);
}
@@ -176,6 +189,9 @@ public class ColumnChunkIncReadStore implements PageReadStore {
for (FSDataInputStream stream : streams) {
stream.close();
}
+ for (ColumnChunkIncPageReader reader : columns.values()) {
+ reader.close();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c3b15af0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a984a4f..457ef3e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -507,7 +507,7 @@
<dependency>
<groupId>de.huxhorn.lilith</groupId>
<artifactId>de.huxhorn.lilith.logback.appender.multiplex-classic</artifactId>
- <version>0.9.43</version>
+ <version>0.9.44</version>
<scope>test</scope>
</dependency>
@@ -804,7 +804,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
- <version>0.94.17-mapr-1405-4.0.0-FCS</version>
+ <version>0.94.17-mapr-1405-m7-4.0.0-FCS</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
[22/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index 406d883..8b4496e 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -117,7 +117,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
@Override
- public int load(int valueCount, ByteBuf buf){
+ public int load(int valueCount, DrillBuf buf){
clear();
this.valueCount = valueCount;
int len = valueCount * ${type.width};
@@ -128,10 +128,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
- assert this.field.matches(metadata);
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
int loaded = load(metadata.getValueCount(), buffer);
- assert metadata.getBufferLength() == loaded;
+ assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
}
public TransferPair getTransferPair(){
@@ -241,23 +241,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
<#if (type.width > 8)>
public ${minor.javaType!type.javaType} get(int index) {
- ByteBuf dst = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.width}]);
- //dst = new io.netty.buffer.SwappedByteBuf(dst);
- data.getBytes(index * ${type.width}, dst, 0, ${type.width});
-
- return dst;
+ return data.slice(index * ${type.width}, ${type.width});
}
<#if (minor.class == "TimeStampTZ")>
public void get(int index, ${minor.class}Holder holder){
holder.value = data.getLong(index * ${type.width});
- holder.index = data.getInt((index * ${type.width})+ ${minor.milliSecondsSize});
+ holder.index = data.getInt((index * ${type.width})+ ${minor.millisecondsSize});
}
public void get(int index, Nullable${minor.class}Holder holder){
holder.isSet = 1;
holder.value = data.getLong(index * ${type.width});
- holder.index = data.getInt((index * ${type.width})+ ${minor.milliSecondsSize});
+ holder.index = data.getInt((index * ${type.width})+ ${minor.millisecondsSize});
}
@Override
@@ -273,7 +269,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
int offsetIndex = index * ${type.width};
holder.months = data.getInt(offsetIndex);
holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
- holder.milliSeconds = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
}
public void get(int index, Nullable${minor.class}Holder holder){
@@ -281,7 +277,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
holder.isSet = 1;
holder.months = data.getInt(offsetIndex);
holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
- holder.milliSeconds = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
}
@Override
@@ -289,7 +285,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
int offsetIndex = index * ${type.width};
int months = data.getInt(offsetIndex);
int days = data.getInt(offsetIndex + ${minor.daysOffset});
- int millis = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
Period p = new Period();
return p.plusMonths(months).plusDays(days).plusMillis(millis);
}
@@ -300,7 +296,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
int months = data.getInt(offsetIndex);
int days = data.getInt(offsetIndex + ${minor.daysOffset});
- int millis = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
@@ -334,20 +330,20 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
int offsetIndex = index * ${type.width};
holder.days = data.getInt(offsetIndex);
- holder.milliSeconds = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
}
public void get(int index, Nullable${minor.class}Holder holder){
int offsetIndex = index * ${type.width};
holder.isSet = 1;
holder.days = data.getInt(offsetIndex);
- holder.milliSeconds = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
}
@Override
public ${friendlyType} getObject(int index) {
int offsetIndex = index * ${type.width};
- int millis = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
int days = data.getInt(offsetIndex);
Period p = new Period();
return p.plusDays(days).plusMillis(millis);
@@ -357,7 +353,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
public StringBuilder getAsStringBuilder(int index) {
int offsetIndex = index * ${type.width};
- int millis = data.getInt(offsetIndex + ${minor.milliSecondsOffset});
+ int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
int days = data.getInt(offsetIndex);
int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
@@ -408,9 +404,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
public ${friendlyType} getObject(int index) {
<#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
// Get the BigDecimal object
- return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
<#else>
- return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
</#if>
}
@@ -428,15 +424,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
@Override
public ${friendlyType} getObject(int index) {
-
- ByteBuf dst = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.width}]);
- //dst = new io.netty.buffer.SwappedByteBuf(dst);
- data.getBytes(index * ${type.width}, dst, 0, ${type.width});
-
- return dst;
-
-
-
+ return data.slice(index * ${type.width}, ${type.width})
}
</#if>
@@ -445,6 +433,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
public ${minor.javaType!type.javaType} get(int index) {
return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
}
+
+ <#if type.width == 4>
+ public long getTwoAsLong(int index) {
+ return data.getLong(index * ${type.width});
+ }
+
+ </#if>
<#if minor.class == "Date">
public ${friendlyType} getObject(int index) {
@@ -513,6 +508,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return get(index);
}
</#if>
+
public void get(int index, ${minor.class}Holder holder){
<#if minor.class.startsWith("Decimal")>
holder.scale = getField().getScale();
@@ -521,7 +517,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
}
-
+
public void get(int index, Nullable${minor.class}Holder holder){
holder.isSet = 1;
holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
@@ -545,7 +541,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
private Mutator(){};
/**
* Set the element at the given index to the given value. Note that widths smaller than
- * 32 bits are handled by the ByteBuf interface.
+ * 32 bits are handled by the DrillBuf interface.
*
* @param index position of the bit to set
* @param value value to set
@@ -565,131 +561,152 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
<#if (minor.class == "TimeStampTZ")>
+ public void set(int index, long timestamp, int tzindex){
+ data.setLong((index * ${type.width}), timestamp);
+ data.setInt(((index * ${type.width}) + ${minor.millisecondsSize}), tzindex);
+ }
+
protected void set(int index, ${minor.class}Holder holder){
- data.setLong((index * ${type.width}), holder.value);
- data.setInt(((index * ${type.width}) + ${minor.milliSecondsSize}), holder.index);
-
+ set(index, holder.value, holder.index);
}
protected void set(int index, Nullable${minor.class}Holder holder){
- data.setLong((index * ${type.width}), holder.value);
- data.setInt(((index * ${type.width}) + ${minor.milliSecondsSize}), holder.index);
+ set(index, holder.value, holder.index);
}
- public boolean setSafe(int index, ${minor.class}Holder holder){
+ public boolean setSafe(int index, long timestamp, int tzindex){
if(index >= getValueCapacity()) {
decrementAllocationMonitor();
return false;
}
- set(index, holder);
+ set(index, timestamp, tzindex);
return true;
}
+
+ public boolean setSafe(int index, ${minor.class}Holder holder){
+ return setSafe(index, holder.value, holder.index);
+ }
public boolean setSafe(int index, Nullable${minor.class}Holder holder){
- if(index >= getValueCapacity()) {
- decrementAllocationMonitor();
- return false;
- }
- set(index, holder);
- return true;
+ return setSafe(index, holder.value, holder.index);
}
+
<#elseif (minor.class == "Interval")>
- protected void set(int index, ${minor.class}Holder holder){
+ public void set(int index, int months, int days, int milliseconds){
int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, holder.months);
- data.setInt((offsetIndex + ${minor.daysOffset}), holder.days);
- data.setInt((offsetIndex + ${minor.milliSecondsOffset}), holder.milliSeconds);
+ data.setInt(offsetIndex, months);
+ data.setInt((offsetIndex + ${minor.daysOffset}), days);
+ data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
}
-
+
+ protected void set(int index, ${minor.class}Holder holder){
+ set(index, holder.months, holder.days, holder.milliseconds);
+ }
+
protected void set(int index, Nullable${minor.class}Holder holder){
- int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, holder.months);
- data.setInt((offsetIndex + ${minor.daysOffset}), holder.days);
- data.setInt((offsetIndex + ${minor.milliSecondsOffset}), holder.milliSeconds);
+ set(index, holder.months, holder.days, holder.milliseconds);
}
- public boolean setSafe(int index, ${minor.class}Holder holder){
+ public boolean setSafe(int index, int months, int days, int milliseconds){
if(index >= getValueCapacity()) {
decrementAllocationMonitor();
return false;
}
- set(index, holder);
+ set(index, months, days, milliseconds);
return true;
}
public boolean setSafe(int index, Nullable${minor.class}Holder holder){
- if(index >= getValueCapacity()) {
- decrementAllocationMonitor();
- return false;
- }
- set(index, holder);
- return true;
+ return setSafe(index, holder.months, holder.days, holder.milliseconds);
+ }
+
+ public boolean setSafe(int index, ${minor.class}Holder holder){
+ return setSafe(index, holder.months, holder.days, holder.milliseconds);
}
+
<#elseif (minor.class == "IntervalDay")>
- protected void set(int index, ${minor.class}Holder holder){
+ public void set(int index, int days, int milliseconds){
int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, holder.days);
- data.setInt((offsetIndex + ${minor.milliSecondsOffset}), holder.milliSeconds);
+ data.setInt(offsetIndex, days);
+ data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+ }
+
+ protected void set(int index, ${minor.class}Holder holder){
+ set(index, holder.days, holder.milliseconds);
}
-
protected void set(int index, Nullable${minor.class}Holder holder){
- int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, holder.days);
- data.setInt((offsetIndex + ${minor.milliSecondsOffset}), holder.milliSeconds);
+ set(index, holder.days, holder.milliseconds);
}
- public boolean setSafe(int index, ${minor.class}Holder holder){
+ public boolean setSafe(int index, int days, int milliseconds){
if(index >= getValueCapacity()) {
decrementAllocationMonitor();
return false;
}
- set(index, holder);
+ set(index, days, milliseconds);
return true;
}
+
+ public boolean setSafe(int index, ${minor.class}Holder holder){
+ return setSafe(index, holder.days, holder.milliseconds);
+ }
public boolean setSafe(int index, Nullable${minor.class}Holder holder){
- if(index >= getValueCapacity()) {
- decrementAllocationMonitor();
- return false;
- }
- set(index, holder);
- return true;
+ return setSafe(index, holder.days, holder.milliseconds);
}
<#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
public void set(int index, ${minor.class}Holder holder){
- data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+ set(index, holder.start, holder.buffer);
}
void set(int index, Nullable${minor.class}Holder holder){
- data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+ set(index, holder.start, holder.buffer);
}
-
+
public boolean setSafe(int index, Nullable${minor.class}Holder holder){
- if(index >= getValueCapacity()) {
- decrementAllocationMonitor();
- return false;
- }
- set(index, holder);
- return true;
+ return setSafe(index, holder.start, holder.buffer);
}
-
public boolean setSafe(int index, ${minor.class}Holder holder){
- if(index >= getValueCapacity()) {
- decrementAllocationMonitor();
- return false;
- }
- set(index, holder);
- return true;
+ return setSafe(index, holder.start, holder.buffer);
+ }
+
+ public boolean setSafe(int index, int start, DrillBuf buffer){
+ if(index >= getValueCapacity()) {
+ decrementAllocationMonitor();
+ return false;
+ }
+ set(index, start, buffer);
+ return true;
+ }
+
+ public void set(int index, int start, DrillBuf buffer){
+ data.setBytes(index * ${type.width}, buffer, start, ${type.width});
}
<#else>
+
protected void set(int index, ${minor.class}Holder holder){
- data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+ set(index, holder.start, holder.buffer);
+ }
+
+ public void set(int index, Nullable${minor.class}Holder holder){
+ set(index, holder.start, holder.buffer);
+ }
+
+ public void set(int index, int start, DrillBuf buffer){
+ data.setBytes(index * ${type.width}, buffer, start, ${type.width});
}
public boolean setSafe(int index, ${minor.class}Holder holder){
+ return setSafe(index, holder.start, holder.buffer);
+ }
+ public boolean setSafe(int index, Nullable${minor.class}Holder holder){
+ return setSafe(index, holder.start, holder.buffer);
+ }
+
+ public boolean setSafe(int index, int start, DrillBuf buffer){
if(index >= getValueCapacity()) {
decrementAllocationMonitor();
return false;
@@ -714,6 +731,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
}
}
+
+
+
<#else> <#-- type.width <= 8 -->
public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
@@ -767,6 +787,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
}
}
+
+
+ public void generateTestDataAlt(int size) {
+ setValueCount(size);
+ boolean even = true;
+ for(int i =0; i < valueCount; i++, even = !even){
+ if(even){
+ set(i, (${(minor.javaType!type.javaType)}) 1);
+ }else{
+ set(i, (${(minor.javaType!type.javaType)}) 0);
+ }
+ }
+ }
</#if> <#-- type.width -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
index bd862a1..7d4ff31 100644
--- a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
+++ b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
@@ -96,15 +96,20 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
@Override
public MajorType getType() {
<#if holderMode == "Repeated">
- return this.repeatedHolder.getType();
+ return this.repeatedHolder.TYPE;
<#else>
- return this.holder.getType();
+ return this.holder.TYPE;
</#if>
}
@Override
public boolean isSet() {
- return this.holder.isSet();
+ <#if nullMode == "Nullable">
+ return this.holder.isSet == 1;
+ <#else>
+ return true;
+ </#if>
+
}
<#if holderMode == "Repeated">
@@ -122,7 +127,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
@Override
public ${friendlyType} read${safeType}(){
<#if nullMode == "Nullable">
- if (!holder.isSet()) {
+ if (!isSet()) {
return null;
}
</#if>
@@ -145,11 +150,11 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Interval">
Period p = new Period();
- return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliSeconds);
+ return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
<#elseif minor.class == "IntervalDay">
Period p = new Period();
- return p.plusDays(holder.days).plusMillis(holder.milliSeconds);
+ return p.plusDays(holder.days).plusMillis(holder.milliseconds);
<#elseif minor.class == "Decimal9" ||
minor.class == "Decimal18" >
@@ -158,7 +163,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Decimal28Dense" ||
minor.class == "Decimal38Dense">
- return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
holder.start,
holder.nDecimalDigits,
holder.scale,
@@ -167,7 +172,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Decimal28Sparse" ||
minor.class == "Decimal38Sparse">
- return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
holder.start,
holder.nDecimalDigits,
holder.scale);
@@ -196,7 +201,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
private Object readSingleObject() {
<#if nullMode == "Nullable">
- if (!holder.isSet()) {
+ if (!isSet()) {
return null;
}
</#if>
@@ -218,11 +223,11 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Interval">
Period p = new Period();
- return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliSeconds);
+ return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
<#elseif minor.class == "IntervalDay">
Period p = new Period();
- return p.plusDays(holder.days).plusMillis(holder.milliSeconds);
+ return p.plusDays(holder.days).plusMillis(holder.milliseconds);
<#elseif minor.class == "Decimal9" ||
minor.class == "Decimal18" >
@@ -231,7 +236,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Decimal28Dense" ||
minor.class == "Decimal38Dense">
- return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
holder.start,
holder.nDecimalDigits,
holder.scale,
@@ -240,7 +245,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Decimal28Sparse" ||
minor.class == "Decimal38Sparse">
- return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
holder.start,
holder.nDecimalDigits,
holder.scale);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/IntervalAggrFunctions2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/IntervalAggrFunctions2.java b/exec/java-exec/src/main/codegen/templates/IntervalAggrFunctions2.java
index ba57210..c7f5918 100644
--- a/exec/java-exec/src/main/codegen/templates/IntervalAggrFunctions2.java
+++ b/exec/java-exec/src/main/codegen/templates/IntervalAggrFunctions2.java
@@ -80,10 +80,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#if type.inputType.endsWith("Interval")>
sum.value += (long) in.months * org.apache.drill.exec.expr.fn.impl.DateUtility.monthToStandardDays +
in.days * (org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis) +
- in.milliSeconds;
+ in.milliseconds;
<#elseif type.inputType.endsWith("IntervalDay")>
sum.value += (long) in.days * (org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis) +
- in.milliSeconds;
+ in.milliseconds;
<#else>
sum.value += in.value;
</#if>
@@ -104,11 +104,11 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
out.months = (int) (millis / org.apache.drill.exec.expr.fn.impl.DateUtility.monthsToMillis);
millis = millis % org.apache.drill.exec.expr.fn.impl.DateUtility.monthsToMillis;
out.days =(int) (millis / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
- out.milliSeconds = (int) (millis % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (millis % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
<#elseif type.inputType.endsWith("IntervalDay")>
out.months = 0;
out.days = (int) (millis / org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
- out.milliSeconds = (int) (millis % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
+ out.milliseconds = (int) (millis % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis);
</#if>
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java
index d1a6d4e..f0ddde2 100644
--- a/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java
@@ -37,7 +37,7 @@ import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter;
import org.apache.drill.exec.store.parquet.ParquetTypeHelper;
import org.apache.drill.exec.vector.*;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
import parquet.io.api.RecordConsumer;
import parquet.schema.MessageType;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index 6876cab..ec99e79 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -47,15 +47,17 @@ package org.apache.drill.exec.vector;
public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector{
private int valueCount;
- final BitVector bits;
+ final UInt1Vector bits;
final ${valuesName} values;
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
+ private final Accessor accessor;
+ private final Mutator mutator;
public ${className}(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
- this.bits = new BitVector(null, allocator);
+ this.bits = new UInt1Vector(null, allocator);
this.values = new ${minor.class}Vector(field, allocator);
+ this.accessor = new Accessor();
+ this.mutator = new Mutator();
}
public int getValueCapacity(){
@@ -71,8 +73,8 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
@Override
- public ByteBuf[] getBuffers() {
- ByteBuf[] buffers = ObjectArrays.concat(bits.getBuffers(), values.getBuffers(), ByteBuf.class);
+ public DrillBuf[] getBuffers() {
+ DrillBuf[] buffers = ObjectArrays.concat(bits.getBuffers(), values.getBuffers(), DrillBuf.class);
clear();
return buffers;
}
@@ -88,7 +90,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
return values.getBufferSize() + bits.getBufferSize();
}
- public ByteBuf getData(){
+ public DrillBuf getData(){
return values.getData();
}
@@ -126,7 +128,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
@Override
- public int load(int dataBytes, int valueCount, ByteBuf buf){
+ public int load(int dataBytes, int valueCount, DrillBuf buf){
clear();
this.valueCount = valueCount;
int loaded = bits.load(valueCount, buf);
@@ -140,10 +142,10 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
- assert this.field.matches(metadata);
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer);
- assert metadata.getBufferLength() == loaded;
+ assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
}
@Override
@@ -199,7 +201,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
@Override
- public int load(int valueCount, ByteBuf buf){
+ public int load(int valueCount, DrillBuf buf){
clear();
this.valueCount = valueCount;
int loaded = bits.load(valueCount, buf);
@@ -211,7 +213,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
+ public void load(SerializedField metadata, DrillBuf buffer) {
assert this.field.matches(metadata);
int loaded = load(metadata.getValueCount(), buffer);
assert metadata.getBufferLength() == loaded;
@@ -321,14 +323,30 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
return b1 && b2;
}
+ public long getDataAddr(){
+ return values.getDataAddr();
+ }
+
+ public long getBitAddr(){
+ return bits.getDataAddr();
+ }
+
+ <#if type.major == "VarLen">
+ public long getOffsetAddr(){
+ return values.getOffsetAddr();
+ }
+ </#if>
public final class Accessor implements ValueVector.Accessor<#if type.major = "VarLen">, VariableWidthVector.VariableWidthAccessor</#if>{
final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
+ final UInt1Vector.Accessor bAccessor = bits.getAccessor();
+ final ${valuesName}.Accessor vAccessor = values.getAccessor();
public FieldReader getReader(){
return reader;
}
+
/**
* Get the element at the specified position.
*
@@ -338,7 +356,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
*/
public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
assert !isNull(index) : "Tried to get null value";
- return values.getAccessor().get(index);
+ return vAccessor.get(index);
}
public boolean isNull(int index) {
@@ -346,8 +364,21 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
public int isSet(int index){
- return bits.getAccessor().get(index);
+ return bAccessor.get(index);
+ }
+
+ <#if type.major == "VarLen">
+ public long getStartEnd(int index){
+ return vAccessor.getStartEnd(index);
+ }
+
+ public long getOffsetAddr(){
+ return values.getOffsetAddr();
}
+ </#if>
+
+
+
<#if type.major == "VarLen">
@@ -357,8 +388,8 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
</#if>
public void get(int index, Nullable${minor.class}Holder holder){
- values.getAccessor().get(index, holder);
- holder.isSet = bits.getAccessor().get(index);
+ vAccessor.get(index, holder);
+ holder.isSet = bAccessor.get(index);
<#if minor.class.startsWith("Decimal")>
holder.scale = getField().getScale();
@@ -371,7 +402,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
if (isNull(index)) {
return null;
}else{
- return values.getAccessor().getObject(index);
+ return vAccessor.getObject(index);
}
}
@@ -380,7 +411,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
if (isNull(index)) {
return null;
}else{
- return values.getAccessor().getAsStringBuilder(index);
+ return vAccessor.getAsStringBuilder(index);
}
}
</#if>
@@ -467,6 +498,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
values.getMutator().set(index, holder);
}
+
public void set(int index, Nullable${minor.class}Holder holder){
<#if type.major == "VarLen">
for (int i = lastSet + 1; i < index; i++) {
@@ -493,14 +525,44 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
return outIndex < Nullable${minor.class}Vector.this.getValueCapacity();
}
- //public boolean setSafe(int index, <#if type.major == "VarLen" || minor.class == "TimeStampTZ" || minor.class == "Interval" || minor.class == "IntervalDay">Nullable${minor.class}Holder <#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value){
+ //public boolean setSafe(int index, int isSet<#if type.major == "VarLen" || minor.class == "TimeStampTZ" || minor.class == "Interval" || minor.class == "IntervalDay">Nullable${minor.class}Holder <#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value){
+
+ <#assign fields = minor.fields!type.fields />
+ public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
+ <#if type.major == "VarLen">
+ for (int i = lastSet + 1; i < index; i++) {
+ values.getMutator().set(i, new byte[]{});
+ }
+ </#if>
+ bits.getMutator().set(index, isSet);
+ values.getMutator().set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ public boolean setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
+ <#if type.major == "VarLen">
+ if(!fillEmpties(index)) return false;
+ </#if>
+
+ boolean b1 = bits.getMutator().setSafe(index, isSet);
+ boolean b2 = values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+ if(b1 && b2){
+ setCount++;
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ return true;
+ }else{
+ return false;
+ }
+
+ }
+
public boolean setSafe(int index, Nullable${minor.class}Holder value) {
<#if type.major == "VarLen">
if(!fillEmpties(index)) return false;
</#if>
- boolean b1 = bits.getMutator().setSafe(index, 1);
+ boolean b1 = bits.getMutator().setSafe(index, value.isSet);
boolean b2 = values.getMutator().setSafe(index, value);
if(b1 && b2){
setCount++;
@@ -561,7 +623,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
public void generateTestData(int valueCount){
- bits.getMutator().generateTestData(valueCount);
+ bits.getMutator().generateTestDataAlt(valueCount);
values.getMutator().generateTestData(valueCount);
<#if type.major = "VarLen">lastSet = valueCount;</#if>
setValueCount(valueCount);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/NumericToCharFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NumericToCharFunctions.java b/exec/java-exec/src/main/codegen/templates/NumericToCharFunctions.java
index 6615e33..ff2c675 100644
--- a/exec/java-exec/src/main/codegen/templates/NumericToCharFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/NumericToCharFunctions.java
@@ -28,6 +28,8 @@ import org.apache.drill.exec.expr.annotations.Workspace;
package org.apache.drill.exec.expr.fn.impl;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -38,6 +40,8 @@ import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import java.text.NumberFormat;
import java.text.DecimalFormat;
@@ -47,12 +51,12 @@ public class G${type}ToChar implements DrillSimpleFunc {
@Param ${type}Holder left;
@Param VarCharHolder right;
- @Workspace ByteBuf buffer;
+ @Inject DrillBuf buffer;
@Workspace java.text.NumberFormat outputFormat;
@Output VarCharHolder out;
public void setup(RecordBatch b) {
- buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[100]);
+ buffer = buffer.reallocIfNeeded(100);
byte[] buf = new byte[right.end - right.start];
right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
String inputFormat = new String(buf);
@@ -65,10 +69,10 @@ public class G${type}ToChar implements DrillSimpleFunc {
java.math.BigDecimal bigDecimal = new java.math.BigDecimal(java.math.BigInteger.valueOf(left.value), left.scale);
String str = outputFormat.format(bigDecimal);
<#elseif type == "Decimal28Sparse" || type == "Decimal38Sparse">
- java.math.BigDecimal bigDecimal = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromByteBuf(left.buffer, left.start, left.nDecimalDigits, left.scale, true);
+ java.math.BigDecimal bigDecimal = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(left.buffer, left.start, left.nDecimalDigits, left.scale, true);
String str = outputFormat.format(bigDecimal);
<#elseif type == "Decimal28Dense" || type == "Decimal38Dense">
- java.math.BigDecimal bigDecimal = org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(left.buffer, left.start, left.nDecimalDigits, left.scale, left.maxPrecision, left.WIDTH);
+ java.math.BigDecimal bigDecimal = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(left.buffer, left.start, left.nDecimalDigits, left.scale, left.maxPrecision, left.WIDTH);
String str = outputFormat.format(bigDecimal);
<#else>
String str = outputFormat.format(left.value);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
index aa25d1a..d4b5af5 100644
--- a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
@@ -36,7 +36,7 @@ import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter;
import org.apache.drill.exec.store.parquet.ParquetTypeHelper;
import org.apache.drill.exec.vector.*;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
import parquet.io.api.RecordConsumer;
import parquet.schema.MessageType;
@@ -191,7 +191,7 @@ public abstract class ParquetOutputRecordWriter extends AbstractRecordWriter imp
byte[] bytes = DecimalUtility.getBigDecimalFromSparse(
holder.buffer, holder.start, ${minor.class}Holder.nDecimalDigits, holder.scale).unscaledValue().toByteArray();
byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.${minor.class?upper_case})];
- if (holder.getSign()) {
+ if (holder.getSign(holder.start, holder.buffer)) {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
} else {
Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 107a667..97117a4 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -78,7 +78,7 @@ package org.apache.drill.exec.vector;
return offsets.getBufferSize() + values.getBufferSize();
}
- public ByteBuf getData(){
+ public DrillBuf getData(){
return values.getData();
}
@@ -199,7 +199,7 @@ package org.apache.drill.exec.vector;
}
@Override
- public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf){
+ public int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf){
clear();
this.parentValueCount = parentValueCount;
this.childValueCount = childValueCount;
@@ -210,10 +210,10 @@ package org.apache.drill.exec.vector;
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
- assert this.field.matches(metadata);
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
int loaded = load(metadata.getVarByteLength(), metadata.getGroupCount(), metadata.getValueCount(), buffer);
- assert metadata.getBufferLength() == loaded;
+ assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
}
public int getByteCapacity(){
@@ -240,7 +240,7 @@ package org.apache.drill.exec.vector;
accessor.reset();
}
- public int load(int parentValueCount, int childValueCount, ByteBuf buf){
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf){
clear();
this.parentValueCount = parentValueCount;
this.childValueCount = childValueCount;
@@ -251,7 +251,7 @@ package org.apache.drill.exec.vector;
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
+ public void load(SerializedField metadata, DrillBuf buffer) {
assert this.field.matches(metadata);
int loaded = load(metadata.getGroupCount(), metadata.getValueCount(), buffer);
assert metadata.getBufferLength() == loaded;
@@ -259,8 +259,8 @@ package org.apache.drill.exec.vector;
</#if>
@Override
- public ByteBuf[] getBuffers() {
- ByteBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(), values.getBuffers(), ByteBuf.class);
+ public DrillBuf[] getBuffers() {
+ DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(), values.getBuffers(), DrillBuf.class);
clear();
return buffers;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
index 701c8b8..b569185 100644
--- a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
+++ b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
@@ -69,7 +69,7 @@ public class ${name}Accessor extends AbstractSqlAccessor{
<#case "VarBinary">
public String getString(int index) {
byte [] b = ac.get(index);
- return DrillStringUtils.toBinaryString(io.netty.buffer.Unpooled.wrappedBuffer(b), 0, b.length);
+ return DrillStringUtils.toBinaryString(b);
}
<#break>
<#case "VarChar">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ValueHolders.java b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
index 46cf9e0..1902843 100644
--- a/exec/java-exec/src/main/codegen/templates/ValueHolders.java
+++ b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
@@ -35,134 +35,76 @@ public final class ${className} implements ValueHolder{
public MajorType getType() {return TYPE;}
- <#if mode.name != "Repeated">
-
+ <#if mode.name == "Repeated">
+
+ /** The first index (inclusive) into the Vector. **/
+ public int start;
+
+ /** The last index (exclusive) into the Vector. **/
+ public int end;
+
+ /** The Vector holding the actual values. **/
+ public ${minor.class}Vector vector;
+
+ <#else>
public static final int WIDTH = ${type.width};
- <#if mode.name == "Optional">
- /** Whether the given holder holds a valid value. 1 means non-null. 0 means null. **/
- public int isSet;
-
- public boolean isSet() {
- return isSet == 1;
- }
- <#else>
- public boolean isSet() {
- return true;
- }
-
- </#if>
-
-
-
- <#if type.major != "VarLen">
-
- <#if (minor.class == "TimeStampTZ")>
- public long value;
- public int index;
- <#elseif (minor.class == "Interval")>
- public int months;
- public int days;
- public int milliSeconds;
- <#elseif (minor.class == "IntervalDay")>
- public int days;
- public int milliSeconds;
- <#elseif minor.class.startsWith("Decimal")>
- public int scale;
- public int precision;
+ <#if mode.name == "Optional">public int isSet;</#if>
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ public ${field.type} ${field.name};
+ </#list>
+
+ <#if minor.class.startsWith("Decimal")>
public static final int maxPrecision = ${minor.maxPrecisionDigits};
<#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
- public int start;
- public ByteBuf buffer;
public static final int nDecimalDigits = ${minor.nDecimalDigits};
-
-
- public int getInteger(int index) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- <#if minor.class.endsWith("Sparse")>
- value = (value & 0x7FFFFFFF);
- <#elseif minor.class.endsWith("Dense")>
- value = (value & 0x0000007F);
- </#if>
- }
- return value;
+
+ public static int getInteger(int index, int start, DrillBuf buffer) {
+ int value = buffer.getInt(start + (index * 4));
+
+ if (index == 0) {
+ /* the first byte contains sign bit, return value without it */
+ <#if minor.class.endsWith("Sparse")>
+ value = (value & 0x7FFFFFFF);
+ <#elseif minor.class.endsWith("Dense")>
+ value = (value & 0x0000007F);
+ </#if>
+ }
+ return value;
}
- public void setInteger(int index, int value) {
+ public static void setInteger(int index, int value, int start, DrillBuf buffer) {
buffer.setInt(start + (index * 4), value);
}
-
- public void setSign(boolean sign) {
+
+ public static void setSign(boolean sign, int start, DrillBuf buffer) {
// Set MSB to 1 if sign is negative
if (sign == true) {
- int value = getInteger(0);
- setInteger(0, (value | 0x80000000));
+ int value = getInteger(0, start, buffer);
+ setInteger(0, (value | 0x80000000), start, buffer);
}
}
-
- public boolean getSign() {
+
+ public static boolean getSign(int start, DrillBuf buffer) {
return ((buffer.getInt(start) & 0x80000000) != 0);
}
-
- <#else>
- public ${minor.javaType!type.javaType} value;
- </#if>
-
- <#elseif (type.width > 8)>
- public int start;
- public ByteBuf buffer;
- <#else>
- public ${minor.javaType!type.javaType} value;
- </#if>
- <#else>
- /** The first offset (inclusive) into the buffer. **/
- public int start;
-
- /** The last offset (exclusive) into the buffer. **/
- public int end;
-
- /** The buffer holding actual values. **/
- public ByteBuf buffer;
-
- public String toString() {
- <#if mode.name == "Optional">
- if (isSet == 0)
- return "<NULL>";
- </#if>
- byte[] buf = new byte[end-start];
- buffer.getBytes(start, buf, 0, end-start);
-
- <#switch minor.class>
- <#case "Var16Char">
- return new String(buf, com.google.common.base.Charsets.UTF_16);
- <#break>
- <#case "VarChar">
- <#default>
- return new String(buf, com.google.common.base.Charsets.UTF_8);
- </#switch>
- }
-
- </#if>
-
- <#else>
+ </#if></#if>
- /** The first index (inclusive) into the Vector. **/
- public int start;
-
- /** The last index (exclusive) into the Vector. **/
- public int end;
-
- /** The Vector holding the actual values. **/
- public ${minor.class}Vector vector;
-
- public boolean isSet() {
- return true;
- }
-
+ @Deprecated
+ public int hashCode(){
+ throw new UnsupportedOperationException();
+ }
+
+ @Deprecated
+ public String toString(){
+ throw new UnsupportedOperationException();
+ }
</#if>
+
+
+
+
}
</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java b/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
index 746d25b..4b9c43b 100644
--- a/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/VarCharAggrFunctions1.java
@@ -33,14 +33,19 @@
package org.apache.drill.exec.expr.fn.impl.gaggr;
+<#include "/@includes/vv_imports.ftl" />
+
import org.apache.drill.exec.expr.DrillAggFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
import org.apache.drill.exec.expr.holders.*;
+import javax.inject.Inject;
import org.apache.drill.exec.record.RecordBatch;
+
import io.netty.buffer.ByteBuf;
@SuppressWarnings("unused")
@@ -57,6 +62,8 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
@Param ${type.inputType}Holder in;
<#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
@Workspace ObjectHolder value;
+ @Workspace UInt1Holder init;
+ @Inject DrillBuf buf;
<#else>
@Workspace ${type.runningType}Holder value;
</#if>
@@ -64,6 +71,8 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
public void setup(RecordBatch b) {
<#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
+ init = new UInt1Holder();
+ init.value = 0;
value = new ObjectHolder();
${type.runningType}Holder tmp = new ${type.runningType}Holder();
tmp.start = 0;
@@ -81,10 +90,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
public void add() {
<#if type.inputType?starts_with("Nullable")>
sout: {
- if (in.isSet == 0) {
- // processing nullable input and the value is null, so don't do anything...
- break sout;
- }
+ if (in.isSet == 0) {
+ // processing nullable input and the value is null, so don't do anything...
+ break sout;
+ }
</#if>
<#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
${type.runningType}Holder tmp = (${type.runningType}Holder)value.obj;
@@ -92,27 +101,12 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
boolean swap = false;
// if buffer is null then swap
- if (tmp.buffer == null) {
+ if (init.value == 0) {
+ init.value = 1;
swap = true;
} else {
// Compare the bytes
- for (int l = in.start, r = tmp.start; l < in.end && r < tmp.end; l++, r++) {
- byte leftByte = in.buffer.getByte(l);
- byte rightByte = tmp.buffer.getByte(r);
- if (leftByte != rightByte) {
- cmp = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
- break;
- }
- }
-
- if (cmp == 0) {
- int l = (in.end - in.start) - (tmp.end - tmp.start);
- if (l > 0) {
- cmp = 1;
- } else {
- cmp = -1;
- }
- }
+ int compare = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.compare(in.buffer.memoryAddress(), in.start, in.end, tmp.buffer.memoryAddress(), tmp.start, tmp.end);
<#if aggrtype.className == "Min">
swap = (cmp == -1);
@@ -122,15 +116,10 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
}
if (swap) {
int length = in.end - in.start;
- if (length > (tmp.end - tmp.start)) {
- // if workspace buffer is smaller, release and allocate a new one
- if (tmp.buffer != null) {
- tmp.buffer.release();
- }
- tmp.buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte [length]);
- }
+ this.buf = tmp.buffer = buf.reallocIfNeeded(in.end - in.start);
in.buffer.getBytes(in.start, tmp.buffer, 0, length);
tmp.end = length;
+
}
<#else>
value.value++;
@@ -156,14 +145,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
public void reset() {
<#if aggrtype.funcName == "max" || aggrtype.funcName == "min">
value = new ObjectHolder();
- ${type.runningType}Holder tmp = new ${type.runningType}Holder();
- tmp.start = 0;
- tmp.end = 0;
- if (tmp.buffer != null) {
- tmp.buffer.release();
- tmp.buffer = null;
- }
- value.obj = tmp;
+ init.value = 0;
<#else>
value = new ${type.runningType}Holder();
value.value = 0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index 0b49f85..95cd3cc 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -40,7 +40,7 @@ package org.apache.drill.exec.vector;
* ${minor.class}Vector implements a vector of variable width values. Elements in the vector
* are accessed by position from the logical start of the vector. A fixed width offsetVector
* is used to convert an element's position to it's offset from the start of the (0-based)
- * ByteBuf. Size is inferred by adjacent elements.
+ * DrillBuf. Size is inferred by adjacent elements.
* The width of each element is ${type.width} byte(s)
* The equivalent Java primitive is '${minor.javaType!type.javaType}'
*
@@ -51,8 +51,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
private final UInt${type.width}Vector offsetVector;
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
+ private final Accessor accessor;
+ private final Mutator mutator;
+
+ private final UInt${type.width}Vector.Accessor oAccessor;
+
private int allocationTotalByteCount = 32768;
private int allocationMonitor = 0;
@@ -60,6 +63,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
this.offsetVector = new UInt${type.width}Vector(null, allocator);
+ this.oAccessor = offsetVector.getAccessor();
+ this.accessor = new Accessor();
+ this.mutator = new Mutator();
}
public int getBufferSize(){
@@ -101,7 +107,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
.build();
}
- public int load(int dataBytes, int valueCount, ByteBuf buf){
+ public int load(int dataBytes, int valueCount, DrillBuf buf){
this.valueCount = valueCount;
if(valueCount == 0){
allocateNew(0,0);
@@ -114,10 +120,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
- assert this.field.matches(metadata);
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer);
- assert metadata.getBufferLength() == loaded;
+ assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
}
@Override
@@ -128,12 +134,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
@Override
- public ByteBuf[] getBuffers() {
- ByteBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(), super.getBuffers(), ByteBuf.class);
+ public DrillBuf[] getBuffers() {
+ DrillBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(), super.getBuffers(), DrillBuf.class);
clear();
return buffers;
}
+ public long getOffsetAddr(){
+ return offsetVector.getDataAddr();
+ }
+
public TransferPair getTransferPair(){
return new TransferImpl(getField());
}
@@ -290,15 +300,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
-
+ final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
public FieldReader getReader(){
return reader;
}
+ public long getStartEnd(int index){
+ return oAccessor.getTwoAsLong(index);
+ }
+
public byte[] get(int index) {
assert index >= 0;
- int startIdx = offsetVector.getAccessor().get(index);
- int length = offsetVector.getAccessor().get(index + 1) - startIdx;
+ int startIdx = oAccessor.get(index);
+ int length = oAccessor.get(index + 1) - startIdx;
assert length >= 0;
byte[] dst = new byte[length];
data.getBytes(startIdx, dst, 0, length);
@@ -310,15 +324,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
}
public void get(int index, ${minor.class}Holder holder){
- holder.start = offsetVector.getAccessor().get(index);
- holder.end = offsetVector.getAccessor().get(index + 1);
+ holder.start = oAccessor.get(index);
+ holder.end = oAccessor.get(index + 1);
holder.buffer = data;
}
public void get(int index, Nullable${minor.class}Holder holder){
holder.isSet = 1;
- holder.start = offsetVector.getAccessor().get(index);
- holder.end = offsetVector.getAccessor().get(index + 1);
+ holder.start = oAccessor.get(index);
+ holder.end = oAccessor.get(index + 1);
holder.buffer = data;
}
@@ -362,7 +376,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
* Mutable${minor.class} implements a vector of variable width values. Elements in the vector
* are accessed by position from the logical start of the vector. A fixed width offsetVector
* is used to convert an element's position to it's offset from the start of the (0-based)
- * ByteBuf. Size is inferred by adjacent elements.
+ * DrillBuf. Size is inferred by adjacent elements.
* The width of each element is ${type.width} byte(s)
* The equivalent Java primitive is '${minor.javaType!type.javaType}'
*
@@ -434,6 +448,26 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
return offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
}
+
+ public boolean setSafe(int index, int start, int end, DrillBuf buffer){
+ int len = end - start;
+
+ int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+ if(data.capacity() < outputStart + len) {
+ decrementAllocationMonitor();
+ return false;
+ }
+
+ if (!offsetVector.getMutator().setSafe( index+1, outputStart + len)) {
+ return false;
+ }
+ buffer.getBytes(start, data, outputStart, len);
+
+ return true;
+ }
+
+
public boolean setSafe(int index, Nullable${minor.class}Holder holder){
assert holder.isSet == 1;
@@ -481,11 +515,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
return true;
}
- protected void set(int index, int start, int length, ByteBuf buffer){
+ protected void set(int index, int start, int length, DrillBuf buffer){
assert index >= 0;
int currentOffset = offsetVector.getAccessor().get(index);
offsetVector.getMutator().set(index + 1, currentOffset + length);
- ByteBuf bb = buffer.slice(start, length);
+ DrillBuf bb = buffer.slice(start, length);
data.setBytes(currentOffset, bb);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java b/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java
deleted file mode 100644
index e5b0f94..0000000
--- a/exec/java-exec/src/main/java/io/netty/buffer/AccountingByteBuf.java
+++ /dev/null
@@ -1,866 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.nio.charset.Charset;
-
-import org.apache.drill.exec.memory.Accountor;
-
-public class AccountingByteBuf extends ByteBuf{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AccountingByteBuf.class);
-
- private final UnsafeDirectLittleEndian b;
- private volatile Accountor acct;
- private volatile int size;
-
- public AccountingByteBuf(Accountor a, ByteBuf b) {
- super();
- this.b = (UnsafeDirectLittleEndian) b;
- this.acct = a;
- this.size = b.capacity();
- }
-
- @Override
- public int refCnt() {
- return b.refCnt();
- }
-
- public boolean transferAccounting(Accountor target){
- boolean outcome = acct.transferTo(target, this, size);
- acct = target;
- return outcome;
- }
-
- @Override
- public synchronized boolean release() {
- if(b.release()){
- acct.release(this, size);
- return true;
- }
- return false;
- }
-
- @Override
- public synchronized boolean release(int decrement) {
- if(b.release(decrement)){
- acct.release(this, size);
- return true;
- }
- return false;
- }
-
- @Override
- public int capacity() {
- return b.capacity();
- }
-
- @Override
- public synchronized ByteBuf capacity(int newCapacity) {
- if(newCapacity == size){
- return this;
- }else if(newCapacity < size){
- b.capacity(newCapacity);
- int diff = size - b.capacity();
- acct.releasePartial(this, diff);
- this.size = size - diff;
- return this;
- }else{
- throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
- }
-
- }
-
- @Override
- public int maxCapacity() {
- return size;
- }
-
- @Override
- public ByteBufAllocator alloc() {
- return b.alloc();
- }
-
- @Override
- public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
- }
-
- @Override
- public ByteBuf order(ByteOrder endianness) {
-// if(endianness != ByteOrder.LITTLE_ENDIAN) throw new UnsupportedOperationException("Drill buffers only support little endian.");
- return this;
- }
-
- @Override
- public ByteBuf unwrap() {
- return this;
- }
-
- @Override
- public boolean isDirect() {
- return true;
- }
-
- @Override
- public int readerIndex() {
- return b.readerIndex();
- }
-
- @Override
- public ByteBuf readerIndex(int readerIndex) {
- b.readerIndex(readerIndex);
- return this;
- }
-
- @Override
- public int writerIndex() {
- return b.writerIndex();
- }
-
- @Override
- public ByteBuf writerIndex(int writerIndex) {
- b.writerIndex(writerIndex);
- return this;
- }
-
- @Override
- public ByteBuf setIndex(int readerIndex, int writerIndex) {
- b.setIndex(readerIndex, writerIndex);
- return this;
- }
-
- @Override
- public int readableBytes() {
- return b.readableBytes();
- }
-
- @Override
- public int writableBytes() {
- return b.writableBytes();
- }
-
- @Override
- public int maxWritableBytes() {
- return b.maxWritableBytes();
- }
-
- @Override
- public boolean isReadable() {
- return b.isReadable();
- }
-
- @Override
- public boolean isReadable(int size) {
- return b.isReadable(size);
- }
-
- @Override
- public boolean isWritable() {
- return b.isWritable();
- }
-
- @Override
- public boolean isWritable(int size) {
- return b.isWritable(size);
- }
-
- @Override
- public ByteBuf clear() {
- b.clear();
- return this;
- }
-
- @Override
- public ByteBuf markReaderIndex() {
- b.markReaderIndex();
- return this;
- }
-
- @Override
- public ByteBuf resetReaderIndex() {
- b.resetReaderIndex();
- return this;
- }
-
- @Override
- public ByteBuf markWriterIndex() {
- b.markWriterIndex();
- return this;
- }
-
- @Override
- public ByteBuf resetWriterIndex() {
- b.resetWriterIndex();
- return this;
- }
-
- @Override
- public ByteBuf discardReadBytes() {
- b.discardReadBytes();
- return this;
- }
-
- @Override
- public ByteBuf discardSomeReadBytes() {
- b.discardSomeReadBytes();
- return this;
- }
-
- @Override
- public ByteBuf ensureWritable(int minWritableBytes) {
- b.ensureWritable(minWritableBytes);
- return this;
- }
-
- @Override
- public int ensureWritable(int minWritableBytes, boolean force) {
- return b.ensureWritable(minWritableBytes, false);
- }
-
- @Override
- public boolean getBoolean(int index) {
- return b.getBoolean(index);
- }
-
- @Override
- public byte getByte(int index) {
- return b.getByte(index);
- }
-
- @Override
- public short getUnsignedByte(int index) {
- return b.getUnsignedByte(index);
- }
-
- @Override
- public short getShort(int index) {
- return b.getShort(index);
- }
-
- @Override
- public int getUnsignedShort(int index) {
- return b.getUnsignedShort(index);
- }
-
- @Override
- public int getMedium(int index) {
- return b.getMedium(index);
- }
-
- @Override
- public int getUnsignedMedium(int index) {
- return b.getUnsignedMedium(index);
- }
-
- @Override
- public int getInt(int index) {
- return b.getInt(index);
- }
-
- @Override
- public long getUnsignedInt(int index) {
- return b.getUnsignedInt(index);
- }
-
- @Override
- public long getLong(int index) {
- return b.getLong(index);
- }
-
- @Override
- public char getChar(int index) {
- return b.getChar(index);
- }
-
- @Override
- public float getFloat(int index) {
- return b.getFloat(index);
- }
-
- @Override
- public double getDouble(int index) {
- return b.getDouble(index);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst) {
- b.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int length) {
- b.getBytes(index, dst, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- b.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst) {
- b.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- b.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- b.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
- b.getBytes(index, out, length);
- return this;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
- return b.getBytes(index, out, length);
- }
-
- @Override
- public ByteBuf setBoolean(int index, boolean value) {
- b.setBoolean(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- b.setByte(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- b.setShort(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setMedium(int index, int value) {
- b.setMedium(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- b.setInt(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- b.setLong(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setChar(int index, int value) {
- b.setChar(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setFloat(int index, float value) {
- b.setFloat(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setDouble(int index, double value) {
- b.setDouble(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src) {
- b.setBytes(index, src);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int length) {
- b.setBytes(index, src, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- b.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src) {
- b.setBytes(index, src);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- b.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- b.setBytes(index, src);
- return this;
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length) throws IOException {
- return b.setBytes(index, in, length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
- return b.setBytes(index, in, length);
- }
-
- @Override
- public ByteBuf setZero(int index, int length) {
- b.setZero(index, length);
- return this;
- }
-
- @Override
- public boolean readBoolean() {
- return b.readBoolean();
- }
-
- @Override
- public byte readByte() {
- return b.readByte();
- }
-
- @Override
- public short readUnsignedByte() {
- return b.readUnsignedByte();
- }
-
- @Override
- public short readShort() {
- return b.readShort();
- }
-
- @Override
- public int readUnsignedShort() {
- return b.readUnsignedShort();
- }
-
- @Override
- public int readMedium() {
- return b.readMedium();
- }
-
- @Override
- public int readUnsignedMedium() {
- return b.readUnsignedMedium();
- }
-
- @Override
- public int readInt() {
- return b.readInt();
- }
-
- @Override
- public long readUnsignedInt() {
- return b.readUnsignedInt();
- }
-
- @Override
- public long readLong() {
- return b.readLong();
- }
-
- @Override
- public char readChar() {
- return b.readChar();
- }
-
- @Override
- public float readFloat() {
- return b.readFloat();
- }
-
- @Override
- public double readDouble() {
- return b.readDouble();
- }
-
- @Override
- public ByteBuf readBytes(int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf readSlice(int length) {
- ByteBuf slice = slice(readerIndex(), length);
- readerIndex(readerIndex() + length);
- return slice;
- }
-
-
- @Override
- public ByteBuf readBytes(ByteBuf dst) {
- b.readBytes(dst);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(ByteBuf dst, int length) {
- b.readBytes(dst, length);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(ByteBuf dst, int dstIndex, int length) {
- b.readBytes(dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(byte[] dst) {
- b.readBytes(dst);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(byte[] dst, int dstIndex, int length) {
- b.readBytes(dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(ByteBuffer dst) {
- b.readBytes(dst);
- return this;
- }
-
- @Override
- public ByteBuf readBytes(OutputStream out, int length) throws IOException {
- b.readBytes(out, length);
- return null;
- }
-
- @Override
- public int readBytes(GatheringByteChannel out, int length) throws IOException {
- return b.readBytes(out, length);
- }
-
- @Override
- public ByteBuf skipBytes(int length) {
- b.skipBytes(length);
- return this;
- }
-
- @Override
- public ByteBuf writeBoolean(boolean value) {
- b.writeBoolean(value);
- return this;
- }
-
- @Override
- public ByteBuf writeByte(int value) {
- b.writeByte(value);
- return this;
- }
-
- @Override
- public ByteBuf writeShort(int value) {
- b.writeShort(value);
- return this;
- }
-
- @Override
- public ByteBuf writeMedium(int value) {
- b.writeMedium(value);
- return this;
- }
-
- @Override
- public ByteBuf writeInt(int value) {
- b.writeInt(value);
- return this;
- }
-
- @Override
- public ByteBuf writeLong(long value) {
- b.writeLong(value);
- return this;
- }
-
- @Override
- public ByteBuf writeChar(int value) {
- b.writeChar(value);
- return this;
- }
-
- @Override
- public ByteBuf writeFloat(float value) {
- b.writeFloat(value);
- return this;
- }
-
- @Override
- public ByteBuf writeDouble(double value) {
- b.writeDouble(value);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuf src) {
- b.writeBytes(src);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuf src, int length) {
- b.writeBytes(src, length);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuf src, int srcIndex, int length) {
- b.writeBytes(src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(byte[] src) {
- b.writeBytes(src);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(byte[] src, int srcIndex, int length) {
- b.writeBytes(src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf writeBytes(ByteBuffer src) {
- b.writeBytes(src);
- return this;
- }
-
- @Override
- public int writeBytes(InputStream in, int length) throws IOException {
- return b.writeBytes(in, length);
- }
-
- @Override
- public int writeBytes(ScatteringByteChannel in, int length) throws IOException {
- return b.writeBytes(in, length);
- }
-
- @Override
- public ByteBuf writeZero(int length) {
- b.writeZero(length);
- return this;
- }
-
- @Override
- public int indexOf(int fromIndex, int toIndex, byte value) {
- return b.indexOf(fromIndex, toIndex, value);
- }
-
- @Override
- public int bytesBefore(byte value) {
- return b.bytesBefore(value);
- }
-
- @Override
- public int bytesBefore(int length, byte value) {
- return b.bytesBefore(length, value);
- }
-
- @Override
- public int bytesBefore(int index, int length, byte value) {
- return b.bytesBefore(index, length, value);
- }
-
- @Override
- public int forEachByte(ByteBufProcessor processor) {
- return b.forEachByte(processor);
- }
-
- @Override
- public int forEachByte(int index, int length, ByteBufProcessor processor) {
- return b.forEachByte(index, length, processor);
- }
-
- @Override
- public int forEachByteDesc(ByteBufProcessor processor) {
- return b.forEachByteDesc(processor);
- }
-
- @Override
- public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
- return b.forEachByteDesc(index, length, processor);
- }
-
- @Override
- public ByteBuf copy() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf slice() {
- return slice(b.readerIndex(), readableBytes());
- }
-
- @Override
- public ByteBuf slice(int index, int length) {
-// if (length == 0) {
-// return Unpooled.EMPTY_BUFFER;
-// }
-
- return new SlicedByteBuf(this, index, length);
- }
-
- @Override
- public ByteBuf duplicate() {
- return new DuplicatedByteBuf(this);
- }
-
- @Override
- public int nioBufferCount() {
- return b.nioBufferCount();
- }
-
- @Override
- public ByteBuffer nioBuffer() {
- return b.nioBuffer();
- }
-
- @Override
- public ByteBuffer nioBuffer(int index, int length) {
- return b.nioBuffer(index, length);
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- return b.internalNioBuffer(index, length);
- }
-
- @Override
- public ByteBuffer[] nioBuffers() {
- return b.nioBuffers();
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return b.nioBuffers(index, length);
- }
-
- @Override
- public boolean hasArray() {
- return b.hasArray();
- }
-
- @Override
- public byte[] array() {
- return b.array();
- }
-
- @Override
- public int arrayOffset() {
- return b.arrayOffset();
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return b.hasMemoryAddress();
- }
-
- @Override
- public long memoryAddress() {
- return b.memoryAddress();
- }
-
- @Override
- public String toString(Charset charset) {
- return b.toString(charset);
- }
-
- @Override
- public String toString(int index, int length, Charset charset) {
- return b.toString(index, length, charset);
- }
-
- @Override
- public int hashCode() {
- return System.identityHashCode(this);
- }
-
- @Override
- public boolean equals(Object obj) {
- // identity equals only.
- return this == obj;
- }
-
- @Override
- public String toString() {
- return "AccountingByteBuf [Inner buffer=" + b + ", size=" + size + "]";
- }
-
- @Override
- public int compareTo(ByteBuf buffer) {
- return b.compareTo(buffer);
- }
-
-
- @Override
- public ByteBuf retain(int increment) {
- b.retain(increment);
- return this;
- }
-
- @Override
- public ByteBuf retain() {
- b.retain();
- return this;
- }
-
-
-}
[08/47] git commit: Fix issue introduced by DRILL-1202 where
allocators are being closed after reporting success. Update ScreenRoot to
cleanup before returning success. Update ScanBatch to cleanup reader in case
of limit query to avoid memory leak in Par
Posted by ja...@apache.org.
Fix issue introduced by DRILL-1202 where allocators are being closed after reporting success.
Update ScreenRoot to cleanup before returning success.
Update ScanBatch to cleanup reader in case of limit query to avoid memory leak in ParquetReader.
Update allocators so that we don't have memory leak when using debug options.
Update project record batch so that it doesn't try to return a released remainder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/15eeb9d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/15eeb9d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/15eeb9d3
Branch: refs/heads/master
Commit: 15eeb9d38af517bc294d6c37c3f7dae82a33665f
Parents: 4216e0e
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Aug 24 10:04:47 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 10:42:50 2014 -0700
----------------------------------------------------------------------
.../drill/exec/memory/TopLevelAllocator.java | 9 +++--
.../apache/drill/exec/ops/FragmentContext.java | 2 +-
.../drill/exec/physical/impl/ScanBatch.java | 4 +++
.../drill/exec/physical/impl/ScreenCreator.java | 21 ++++++++----
.../impl/project/ProjectRecordBatch.java | 8 +++++
.../exec/work/fragment/FragmentExecutor.java | 36 +++++++++++++++-----
.../org/apache/drill/TestExampleQueries.java | 13 +++++++
7 files changed, 73 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 098922c..d89c892 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -98,7 +98,7 @@ public class TopLevelAllocator implements BufferAllocator {
throw new OutOfMemoryException(String.format("You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.", initialReservation, acct.getCapacity() - acct.getAllocation()));
};
logger.debug("New child allocator with initial reservation {}", initialReservation);
- ChildAllocator allocator = new ChildAllocator(handle, acct, maximumReservation, initialReservation);
+ ChildAllocator allocator = new ChildAllocator(handle, acct, maximumReservation, initialReservation, childrenMap);
if(ENABLE_ACCOUNTING) childrenMap.put(allocator, Thread.currentThread().getStackTrace());
return allocator;
@@ -130,11 +130,13 @@ public class TopLevelAllocator implements BufferAllocator {
private Map<ChildAllocator, StackTraceElement[]> children = new HashMap<>();
private boolean closed = false;
private FragmentHandle handle;
+ private Map<ChildAllocator, StackTraceElement[]> thisMap;
- public ChildAllocator(FragmentHandle handle, Accountor parentAccountor, long max, long pre) throws OutOfMemoryException{
+ public ChildAllocator(FragmentHandle handle, Accountor parentAccountor, long max, long pre, Map<ChildAllocator, StackTraceElement[]> map) throws OutOfMemoryException{
assert max >= pre;
childAcct = new Accountor(errorOnLeak, handle, parentAccountor, max, pre);
this.handle = handle;
+ thisMap = map;
}
@Override
@@ -171,7 +173,7 @@ public class TopLevelAllocator implements BufferAllocator {
throw new OutOfMemoryException(String.format("You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.", initialReservation, childAcct.getAvailable()));
};
logger.debug("New child allocator with initial reservation {}", initialReservation);
- ChildAllocator newChildAllocator = new ChildAllocator(handle, childAcct, maximumReservation, initialReservation);
+ ChildAllocator newChildAllocator = new ChildAllocator(handle, childAcct, maximumReservation, initialReservation, null);
this.children.put(newChildAllocator, Thread.currentThread().getStackTrace());
return newChildAllocator;
}
@@ -183,6 +185,7 @@ public class TopLevelAllocator implements BufferAllocator {
@Override
public void close() {
if (ENABLE_ACCOUNTING) {
+ if(thisMap != null) thisMap.remove(this);
for (ChildAllocator child : children.keySet()) {
if (!child.isClosed()) {
StringBuilder sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 8a670fb..6fccb3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -111,7 +111,7 @@ public class FragmentContext implements Closeable {
}
public void fail(Throwable cause) {
- logger.error("Fragment Context received failure. {}", cause);
+ logger.error("Fragment Context received failure.", cause);
failed = true;
failureCause = cause;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 4b7e33e..9472627 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -129,6 +129,10 @@ public class ScanBatch implements RecordBatch {
@Override
public void kill(boolean sendUpstream) {
+ if(currentReader != null){
+ currentReader.cleanup();
+ }
+
if (sendUpstream) {
done = true;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
index 279374f..d96bdf3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
@@ -99,7 +99,7 @@ public class ScreenCreator implements RootCreator<Screen>{
// logger.debug("Screen Outcome {}", outcome);
switch(outcome){
case STOP: {
- sendCount.waitForSendComplete();
+ this.internalStop();
boolean verbose = context.getOptions().getOption(ExecConstants.ENABLE_VERBOSE_ERRORS_KEY).bool_val;
QueryResult header = QueryResult.newBuilder() //
.setQueryId(context.getHandle().getQueryId()) //
@@ -122,8 +122,7 @@ public class ScreenCreator implements RootCreator<Screen>{
return false;
}
case NONE: {
- sendCount.waitForSendComplete();
-// context.getStats().batchesCompleted.inc(1);
+ this.internalStop();
QueryWritableBatch batch;
if (!first) {
QueryResult header = QueryResult.newBuilder() //
@@ -168,22 +167,30 @@ public class ScreenCreator implements RootCreator<Screen>{
throw new UnsupportedOperationException();
}
}
-
+
public void updateStats(QueryWritableBatch queryBatch) {
stats.addLongStat(Metric.BYTES_SENT, queryBatch.getByteCount());
}
- @Override
- public void stop() {
+
+ private void internalStop(){
sendCount.waitForSendComplete();
oContext.close();
incoming.cleanup();
}
+ @Override
+ public void stop() {
+ if(!oContext.isClosed()){
+ internalStop();
+ }
+ sendCount.waitForSendComplete();
+ }
+
private SendListener listener = new SendListener();
private class SendListener extends BaseRpcOutcomeListener<Ack>{
- volatile RpcException ex;
+ volatile RpcException ex;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 4fdb71d..a7e1360 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -111,6 +111,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
return recordCount;
}
+
+ @Override
+ protected void killIncoming(boolean sendUpstream) {
+ super.killIncoming(sendUpstream);
+ hasRemainder = false;
+ }
+
+
@Override
public IterOutcome innerNext() {
if (hasRemainder) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index e4941d0..2a8db67 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -17,9 +17,9 @@
*/
package org.apache.drill.exec.work.fragment;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.FragmentRoot;
import org.apache.drill.exec.physical.impl.ImplCreator;
@@ -31,8 +31,6 @@ import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.work.CancelableQuery;
import org.apache.drill.exec.work.StatusProvider;
-
-import com.codahale.metrics.Timer;
import org.apache.drill.exec.work.WorkManager.WorkerBee;
/**
@@ -49,6 +47,7 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid
private final WorkerBee bee;
private final StatusReporter listener;
private Thread executionThread;
+ private AtomicBoolean closed = new AtomicBoolean(false);
public FragmentExecutor(FragmentContext context, WorkerBee bee, FragmentRoot rootOperator, StatusReporter listener){
this.context = context;
@@ -104,10 +103,14 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid
while (state.get() == FragmentState.RUNNING_VALUE) {
if (!root.next()) {
if (context.isFailed()){
- updateState(FragmentState.RUNNING, FragmentState.FAILED, false);
+ internalFail(context.getFailureCause());
+ closeOutResources(false);
} else {
+ closeOutResources(true); // make sure to close out resources before we report success.
updateState(FragmentState.RUNNING, FragmentState.FINISHED, false);
}
+
+ break;
}
}
} catch (AssertionError | Exception e) {
@@ -116,17 +119,32 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid
internalFail(e);
} finally {
bee.removeFragment(context.getHandle());
- if (context.isFailed()) {
- internalFail(context.getFailureCause());
- }
- root.stop(); // stop root executor & clean-up resources
- context.close();
logger.debug("Fragment runner complete. {}:{}", context.getHandle().getMajorFragmentId(), context.getHandle().getMinorFragmentId());
Thread.currentThread().setName(originalThread);
}
}
+ private void closeOutResources(boolean throwFailure){
+ if(closed.get()) return;
+
+ try{
+ root.stop();
+ }catch(RuntimeException e){
+ if(throwFailure) throw e;
+ logger.warn("Failure while closing out resources.", e);
+ }
+
+ try{
+ context.close();
+ }catch(RuntimeException e){
+ if(throwFailure) throw e;
+ logger.warn("Failure while closing out resources.", e);
+ }
+
+ closed.set(true);
+ }
+
private void internalFail(Throwable excep){
state.set(FragmentState.FAILED_VALUE);
listener.fail(context.getHandle(), "Failure while running fragment.", excep);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/15eeb9d3/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 0ac2c09..d9e8b20 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -47,6 +47,19 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test
+ public void testJoinMerge() throws Exception{
+ test("alter session set `planner.enable_hashjoin` = false");
+ test("select count(*) \n" +
+ " from (select l.l_orderkey as x, c.c_custkey as y \n" +
+ " from cp.`tpch/lineitem.parquet` l \n" +
+ " left outer join cp.`tpch/customer.parquet` c \n" +
+ " on l.l_orderkey = c.c_custkey) as foo\n" +
+ " where x < 10000\n" +
+ "");
+ test("alter session set `planner.enable_hashjoin` = true");
+ }
+
+ @Test
public void testSelStarOrderBy() throws Exception{
test("select * from cp.`employee.json` order by last_name");
}
[29/47] git commit: Fix issues with Hive function generation to
support DrillBuf
Posted by ja...@apache.org.
Fix issues with Hive function generation to support DrillBuf
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/91870359
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/91870359
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/91870359
Branch: refs/heads/master
Commit: 91870359947b8457f4613953fe59948fa9f2fe17
Parents: b5bbed8
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Aug 25 21:49:46 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Aug 25 21:49:46 2014 -0700
----------------------------------------------------------------------
.../codegen/templates/ObjectInspectorHelper.java | 17 ++++++++++++-----
.../main/codegen/templates/ObjectInspectors.java | 17 +++++++++++++----
.../apache/drill/exec/expr/fn/HiveFuncHolder.java | 2 +-
3 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/91870359/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
index 091f027..961dfbb 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
@@ -28,6 +28,7 @@ import com.sun.codemodel.*;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.DirectExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -73,7 +74,7 @@ public class ObjectInspectorHelper {
String.format("Type %s[%s] not supported as arguement to Hive UDFs", minorType.toString(), mode.toString()));
}
- public static JBlock initReturnValueHolder(JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) {
+ public static JBlock initReturnValueHolder(ClassGenerator<?> g, JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) {
JBlock block = new JBlock(false, false);
switch(oi.getCategory()) {
case PRIMITIVE: {
@@ -85,10 +86,16 @@ public class ObjectInspectorHelper {
block.assign(returnValueHolder, JExpr._new(holderClass));
<#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING" || entry.hiveType == "BINARY">
- block.assign(returnValueHolder.ref("buffer"),
- m.directClass(io.netty.buffer.Unpooled.class.getCanonicalName())
- .staticInvoke("wrappedBuffer")
- .arg(JExpr.newArray(m._ref(byte.class), JExpr.lit(1000))));
+ block.assign( //
+ returnValueHolder.ref("buffer"), //
+ g
+ .getMappingSet()
+ .getIncoming()
+ .invoke("getContext")
+ .invoke("getManagedBuffer")
+ .invoke("reallocIfNeeded")
+ .arg(JExpr.lit(1024))
+ );
</#if>
return block;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/91870359/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
index 576fa47..3345b75 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
@@ -25,6 +25,7 @@
package org.apache.drill.exec.expr.fn.impl.hive;
import org.apache.drill.exec.util.DecimalUtility;
+import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers;
import org.apache.drill.exec.expr.holders.*;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
@@ -62,14 +63,18 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri
public static class Required extends Drill${entry.drillType}ObjectInspector {
@Override
public HiveVarchar getPrimitiveJavaObject(Object o) {
- return new HiveVarchar(((VarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
+ VarCharHolder h = (VarCharHolder)o;
+ String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer);
+ return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH);
}
}
public static class Optional extends Drill${entry.drillType}ObjectInspector {
@Override
public HiveVarchar getPrimitiveJavaObject(Object o) {
- return new HiveVarchar(((NullableVarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
+ NullableVarCharHolder h = (NullableVarCharHolder)o;
+ String s = h.isSet == 0 ? null : StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer);
+ return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH);
}
}
@@ -82,14 +87,18 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri
public static class Required extends Drill${entry.drillType}ObjectInspector {
@Override
public String getPrimitiveJavaObject(Object o){
- return((Var16CharHolder)o).toString();
+ Var16CharHolder h = (Var16CharHolder)o;
+ String s = StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer);
+ return s;
}
}
public static class Optional extends Drill${entry.drillType}ObjectInspector {
@Override
public String getPrimitiveJavaObject(Object o){
- return((NullableVar16CharHolder)o).toString();
+ NullableVar16CharHolder h = (NullableVar16CharHolder)o;
+ String s = h.isSet == 0 ? null : StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer);
+ return s;
}
}
<#elseif entry.drillType == "VarBinary">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/91870359/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
index 45dbafe..2bc3576 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
@@ -218,7 +218,7 @@ public class HiveFuncHolder extends AbstractFuncHolder {
._throw(JExpr._new(m.directClass(RuntimeException.class.getCanonicalName()))
.arg(JExpr.lit(String.format("Failed to initialize GenericUDF"))).arg(exVar));
- sub.add(ObjectInspectorHelper.initReturnValueHolder(m, workspaceJVars[4], returnOI, returnType.getMinorType()));
+ sub.add(ObjectInspectorHelper.initReturnValueHolder(g, m, workspaceJVars[4], returnOI, returnType.getMinorType()));
// now add it to the doSetup block in Generated class
JBlock setup = g.getBlock(ClassGenerator.BlockType.SETUP);
[47/47] git commit: Update to latest parquet zero copy version.
Posted by ja...@apache.org.
Update to latest parquet zero copy version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c1c0eba5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c1c0eba5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c1c0eba5
Branch: refs/heads/master
Commit: c1c0eba5bad30a433161ebf4e1b6c36ace2a881e
Parents: 46de881
Author: Jacques Nadeau <ja...@apache.org>
Authored: Thu Aug 28 12:49:44 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Aug 28 12:49:44 2014 -0700
----------------------------------------------------------------------
exec/java-exec/pom.xml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c1c0eba5/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index d1f9754..e15bc77 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -131,7 +131,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-column</artifactId>
- <version>1.5.1-drill-r3</version>
+ <version>1.5.1-drill-r4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -146,7 +146,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
- <version>1.5.1-drill-r3</version>
+ <version>1.5.1-drill-r4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -176,7 +176,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-common</artifactId>
- <version>1.5.1-drill-r3</version>
+ <version>1.5.1-drill-r4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -191,7 +191,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-jackson</artifactId>
- <version>1.5.1-drill-r3</version>
+ <version>1.5.1-drill-r4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -206,7 +206,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-encoding</artifactId>
- <version>1.5.1-drill-r3</version>
+ <version>1.5.1-drill-r4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -221,7 +221,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-generator</artifactId>
- <version>1.5.1-drill-r3</version>
+ <version>1.5.1-drill-r4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
[09/47] git commit: Turn off proguard verbose output.
Posted by ja...@apache.org.
Turn off proguard verbose output.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/35296501
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/35296501
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/35296501
Branch: refs/heads/master
Commit: 3529650119a0096f1b98748ace7c800d019fe905
Parents: 15eeb9d
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Aug 24 10:45:01 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 10:45:01 2014 -0700
----------------------------------------------------------------------
exec/jdbc-all/pom.xml | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/35296501/exec/jdbc-all/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 2a045ac..d767977 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -207,7 +207,6 @@
<outputDirectory>${project.build.directory}</outputDirectory>
<maxMemory>6g</maxMemory>
<options>
- <option>-verbose</option>
<option>-dontobfuscate</option>
<option>-dontoptimize</option>
<option>-ignorewarnings</option>
[37/47] git commit: DRILL-998: Limit amount of memory used by drill
C++ client API
Posted by ja...@apache.org.
DRILL-998: Limit amount of memory used by drill C++ client API
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5a7feb9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5a7feb9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5a7feb9d
Branch: refs/heads/master
Commit: 5a7feb9dca73bc11249210e7a241a11533f6944e
Parents: cd9eaa8
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Aug 27 10:31:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 13:33:48 2014 -0700
----------------------------------------------------------------------
.../native/client/example/querySubmitter.cpp | 1 +
.../native/client/src/clientlib/CMakeLists.txt | 1 +
.../native/client/src/clientlib/drillClient.cpp | 11 ++--
.../client/src/clientlib/drillClientImpl.cpp | 50 ++++++++------
.../client/src/clientlib/drillClientImpl.hpp | 12 ++--
.../native/client/src/clientlib/recordBatch.cpp | 2 +-
contrib/native/client/src/clientlib/utils.cpp | 68 ++++++++++++++++++++
contrib/native/client/src/clientlib/utils.hpp | 31 ++++++---
.../native/client/src/include/drill/common.hpp | 6 ++
.../client/src/include/drill/recordBatch.hpp | 8 +--
10 files changed, 147 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/example/querySubmitter.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/example/querySubmitter.cpp b/contrib/native/client/example/querySubmitter.cpp
index 8e80658..17bec3c 100644
--- a/contrib/native/client/example/querySubmitter.cpp
+++ b/contrib/native/client/example/querySubmitter.cpp
@@ -280,6 +280,7 @@ int main(int argc, char* argv[]) {
//DrillClient::initLogging("/var/log/drill/", l);
// To log to stderr
Drill::DrillClient::initLogging(NULL, l);
+ Drill::DrillClientConfig::setBufferLimit(2*1024*1024); // 2MB. Allows us to hold at least two record batches.
if(client.connect(connectStr.c_str(), schema.c_str())!=Drill::CONN_SUCCESS){
std::cerr<< "Failed to connect with error: "<< client.getError() << " (Using:"<<connectStr<<")"<<std::endl;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/CMakeLists.txt b/contrib/native/client/src/clientlib/CMakeLists.txt
index 37f4734..dc8f032 100644
--- a/contrib/native/client/src/clientlib/CMakeLists.txt
+++ b/contrib/native/client/src/clientlib/CMakeLists.txt
@@ -27,6 +27,7 @@ set (CLIENTLIB_SRC_FILES
${CMAKE_CURRENT_SOURCE_DIR}/rpcDecoder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/errmsgs.cpp
${CMAKE_CURRENT_SOURCE_DIR}/logger.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp
)
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../include )
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/drillClient.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp
index a7489bc..6611332 100644
--- a/contrib/native/client/src/clientlib/drillClient.cpp
+++ b/contrib/native/client/src/clientlib/drillClient.cpp
@@ -43,7 +43,7 @@ DrillClientInitializer::~DrillClientInitializer(){
}
logLevel_t DrillClientConfig::s_logLevel=LOG_ERROR;
-uint64_t DrillClientConfig::s_bufferLimit=-1;
+uint64_t DrillClientConfig::s_bufferLimit=MAX_MEM_ALLOC_SIZE;
int32_t DrillClientConfig::s_socketTimeout=180;
boost::mutex DrillClientConfig::s_mutex;
@@ -157,10 +157,6 @@ FieldDefPtr RecordIterator::getColDefs(){
status_t RecordIterator::next(){
status_t ret=QRY_SUCCESS;
- this->m_pQueryResult->waitForData();
- if(m_pQueryResult->hasError()){
- return m_pQueryResult->getErrorStatus();
- }
this->m_currentRecord++;
if(!this->m_pQueryResult->isCancelled()){
@@ -169,8 +165,13 @@ status_t RecordIterator::next(){
if(this->m_pCurrentRecordBatch !=NULL){
DRILL_LOG(LOG_TRACE) << "Deleted old Record batch " << (void*) m_pCurrentRecordBatch << std::endl;
delete this->m_pCurrentRecordBatch; //free the previous record batch
+ this->m_pCurrentRecordBatch=NULL;
}
this->m_currentRecord=0;
+ this->m_pQueryResult->waitForData();
+ if(m_pQueryResult->hasError()){
+ return m_pQueryResult->getErrorStatus();
+ }
this->m_pCurrentRecordBatch=this->m_pQueryResult->getNext();
if(this->m_pCurrentRecordBatch != NULL){
DRILL_LOG(LOG_TRACE) << "Fetched new Record batch " << std::endl;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/drillClientImpl.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp
index 54dcdd0..0ea4897 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.cpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp
@@ -175,7 +175,7 @@ connectionStatus_t DrillClientImpl::recvHandshake(){
DRILL_LOG(LOG_DEBUG) << "Sent handshake read request to server" << std::endl;
m_io_service.run();
if(m_rbuf!=NULL){
- Utils::freeBuffer(m_rbuf); m_rbuf=NULL;
+ Utils::freeBuffer(m_rbuf, MAX_SOCK_RD_BUFSIZE); m_rbuf=NULL;
}
return CONN_SUCCESS;
}
@@ -332,6 +332,13 @@ void DrillClientImpl::getNextResult(){
// This call is always made from within a function where the mutex has already been acquired
//boost::lock_guard<boost::mutex> lock(this->m_dcMutex);
+ {
+ boost::unique_lock<boost::mutex> memLock(AllocatedBuffer::s_memCVMutex);
+ DRILL_LOG(LOG_TRACE) << "Read blocked waiting for memory." << std::endl;
+ while(AllocatedBuffer::s_isBufferLimitReached){
+ AllocatedBuffer::s_memCV.wait(memLock);
+ }
+ }
//use free, not delete to free
ByteBuf_t readBuf = Utils::allocateBuffer(LEN_PREFIX_BUFLEN);
@@ -362,10 +369,13 @@ void DrillClientImpl::waitForResults(){
delete this->m_pListenerThread; this->m_pListenerThread=NULL;
}
-status_t DrillClientImpl::readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, InBoundRpcMessage& msg, boost::system::error_code& error){
+status_t DrillClientImpl::readMsg(ByteBuf_t _buf,
+ AllocatedBufferPtr* allocatedBuffer,
+ InBoundRpcMessage& msg,
+ boost::system::error_code& error){
size_t leftover=0;
uint32_t rmsgLen;
- ByteBuf_t currentBuffer;
+ AllocatedBufferPtr currentBuffer;
*allocatedBuffer=NULL;
{
// We need to protect the readLength and read buffer, and the pending requests counter,
@@ -379,18 +389,18 @@ status_t DrillClientImpl::readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, In
leftover = LEN_PREFIX_BUFLEN - bytes_read;
// Allocate a buffer
DRILL_LOG(LOG_TRACE) << "Allocated and locked buffer." << std::endl;
- currentBuffer=Utils::allocateBuffer(rmsgLen);
+ currentBuffer=new AllocatedBuffer(rmsgLen);
if(currentBuffer==NULL){
- Utils::freeBuffer(_buf);
+ Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN);
return handleQryError(QRY_CLIENT_OUTOFMEM, getMessage(ERR_QRY_OUTOFMEM), NULL);
}
*allocatedBuffer=currentBuffer;
if(leftover){
- memcpy(currentBuffer, _buf + bytes_read, leftover);
+ memcpy(currentBuffer->m_pBuffer, _buf + bytes_read, leftover);
}
DRILL_LOG(LOG_TRACE) << "reading data (rmsgLen - leftover) : "
<< (rmsgLen - leftover) << std::endl;
- ByteBuf_t b=currentBuffer + leftover;
+ ByteBuf_t b=currentBuffer->m_pBuffer + leftover;
size_t bytesToRead=rmsgLen - leftover;
while(1){
size_t dataBytesRead=this->m_socket.read_some(
@@ -404,24 +414,24 @@ status_t DrillClientImpl::readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, In
}
if(!error){
// read data successfully
- DrillClientImpl::s_decoder.Decode(currentBuffer, rmsgLen, msg);
+ DrillClientImpl::s_decoder.Decode(currentBuffer->m_pBuffer, rmsgLen, msg);
DRILL_LOG(LOG_TRACE) << "Done decoding chunk. Coordination id: " <<msg.m_coord_id<< std::endl;
}else{
- Utils::freeBuffer(_buf);
+ Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN);
return handleQryError(QRY_COMM_ERROR,
getMessage(ERR_QRY_COMMERR, error.message().c_str()), NULL);
}
}else{
// got a message with an invalid read length.
- Utils::freeBuffer(_buf);
+ Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN);
return handleQryError(QRY_INTERNAL_ERROR, getMessage(ERR_QRY_INVREADLEN), NULL);
}
}
- Utils::freeBuffer(_buf);
+ Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN);
return QRY_SUCCESS;
}
-status_t DrillClientImpl::processQueryResult(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg ){
+status_t DrillClientImpl::processQueryResult(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg ){
DrillClientQueryResult* pDrillClientQueryResult=NULL;
status_t ret=QRY_SUCCESS;
{
@@ -453,14 +463,14 @@ status_t DrillClientImpl::processQueryResult(ByteBuf_t allocatedBuffer, InBoundR
//Check QueryResult.queryState. QueryResult could have an error.
if(qr->query_state() == exec::shared::QueryResult_QueryState_FAILED){
status_t ret=handleQryError(QRY_FAILURE, qr->error(0), pDrillClientQueryResult);
- Utils::freeBuffer(allocatedBuffer);
+ delete allocatedBuffer;
delete qr;
return ret;
}
//Validate the RPC message
std::string valErr;
if( (ret=validateMessage(msg, *qr, valErr)) != QRY_SUCCESS){
- Utils::freeBuffer(allocatedBuffer);
+ delete allocatedBuffer;
delete qr;
return handleQryError(ret, getMessage(ERR_QRY_INVRPC, valErr.c_str()), pDrillClientQueryResult);
}
@@ -521,7 +531,7 @@ status_t DrillClientImpl::processQueryResult(ByteBuf_t allocatedBuffer, InBoundR
return ret;
}
-status_t DrillClientImpl::processQueryId(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg ){
+status_t DrillClientImpl::processQueryId(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg ){
DrillClientQueryResult* pDrillClientQueryResult=NULL;
DRILL_LOG(LOG_DEBUG) << "Processing Query Handle with coordination id:" << msg.m_coord_id << std::endl;
status_t ret=QRY_SUCCESS;
@@ -539,10 +549,10 @@ status_t DrillClientImpl::processQueryId(ByteBuf_t allocatedBuffer, InBoundRpcMe
//save queryId allocated here so we can free it later
pDrillClientQueryResult->setQueryId(qid);
}else{
- Utils::freeBuffer(allocatedBuffer);
+ delete allocatedBuffer;
return handleQryError(QRY_INTERNAL_ERROR, getMessage(ERR_QRY_INVQUERYID), NULL);
}
- Utils::freeBuffer(allocatedBuffer);
+ delete allocatedBuffer;
return ret;
}
@@ -580,7 +590,7 @@ void DrillClientImpl::handleRead(ByteBuf_t _buf,
InBoundRpcMessage msg;
DRILL_LOG(LOG_TRACE) << "Getting new message" << std::endl;
- ByteBuf_t allocatedBuffer=NULL;
+ AllocatedBufferPtr allocatedBuffer=NULL;
if(readMsg(_buf, &allocatedBuffer, msg, error)!=QRY_SUCCESS){
if(m_pendingRequests!=0){
@@ -628,7 +638,7 @@ void DrillClientImpl::handleRead(ByteBuf_t _buf,
}
}else{
// boost error
- Utils::freeBuffer(_buf);
+ Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN);
boost::lock_guard<boost::mutex> lock(this->m_dcMutex);
handleQryError(QRY_COMM_ERROR, getMessage(ERR_QRY_COMMERR, error.message().c_str()), NULL);
return;
@@ -828,7 +838,7 @@ status_t DrillClientQueryResult::defaultQueryResultsListener(void* ctx,
return QRY_SUCCESS;
}
-RecordBatch* DrillClientQueryResult::peekNext() {
+RecordBatch* DrillClientQueryResult::peekNext(){
RecordBatch* pRecordBatch=NULL;
//if no more data, return NULL;
if(!m_bIsQueryPending) return NULL;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/drillClientImpl.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp
index e40b214..3ac0b20 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.hpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp
@@ -198,7 +198,7 @@ class DrillClientImpl{
m_socket.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ignorederr);
m_socket.close();
if(m_rbuf!=NULL){
- Utils::freeBuffer(m_rbuf); m_rbuf=NULL;
+ Utils::freeBuffer(m_rbuf, MAX_SOCK_RD_BUFSIZE); m_rbuf=NULL;
}
if(m_pError!=NULL){
delete m_pError; m_pError=NULL;
@@ -244,9 +244,13 @@ class DrillClientImpl{
void handleHShakeReadTimeout(const boost::system::error_code & err);
// Query results
void getNextResult();
- status_t readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, InBoundRpcMessage& msg, boost::system::error_code& error);
- status_t processQueryResult(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg);
- status_t processQueryId(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg );
+ status_t readMsg(
+ ByteBuf_t _buf,
+ AllocatedBufferPtr* allocatedBuffer,
+ InBoundRpcMessage& msg,
+ boost::system::error_code& error);
+ status_t processQueryResult(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg);
+ status_t processQueryId(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg );
void handleReadTimeout(const boost::system::error_code & err);
void handleRead(ByteBuf_t _buf, const boost::system::error_code & err, size_t bytes_transferred) ;
status_t validateMessage(InBoundRpcMessage& msg, exec::shared::QueryResult& qr, std::string& valError);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/recordBatch.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/recordBatch.cpp b/contrib/native/client/src/clientlib/recordBatch.cpp
index 17073bd..4c55f04 100644
--- a/contrib/native/client/src/clientlib/recordBatch.cpp
+++ b/contrib/native/client/src/clientlib/recordBatch.cpp
@@ -312,7 +312,7 @@ RecordBatch::~RecordBatch(){
}
m_fieldDefs->clear();
delete m_pQueryResult;
- Utils::freeBuffer(m_allocatedBuffer);
+ delete m_allocatedBuffer;
}
ret_t RecordBatch::build(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/utils.cpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/utils.cpp b/contrib/native/client/src/clientlib/utils.cpp
new file mode 100644
index 0000000..f1f03a1
--- /dev/null
+++ b/contrib/native/client/src/clientlib/utils.cpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "utils.hpp"
+#include "drill/common.hpp"
+
+namespace Drill{
+
+
+boost::mutex AllocatedBuffer::s_memCVMutex;
+boost::condition_variable AllocatedBuffer::s_memCV;
+size_t AllocatedBuffer::s_allocatedMem=0;
+bool AllocatedBuffer::s_isBufferLimitReached=false;
+
+ByteBuf_t Utils::allocateBuffer(size_t len){
+ boost::lock_guard<boost::mutex> memLock(AllocatedBuffer::s_memCVMutex);
+ AllocatedBuffer::s_allocatedMem+=len;
+ //http://stackoverflow.com/questions/2688466/why-mallocmemset-is-slower-than-calloc
+ ByteBuf_t b = (ByteBuf_t)calloc(len, sizeof(Byte_t));
+ size_t safeSize= DrillClientConfig::getBufferLimit()-MEM_CHUNK_SIZE;
+ if(b!=NULL && AllocatedBuffer::s_allocatedMem >= safeSize){
+ AllocatedBuffer::s_isBufferLimitReached=true;
+ }
+ return b;
+}
+
+void Utils::freeBuffer(ByteBuf_t b, size_t len){
+ boost::lock_guard<boost::mutex> memLock(AllocatedBuffer::s_memCVMutex);
+ AllocatedBuffer::s_allocatedMem-=len;
+ free(b);
+ size_t safeSize= DrillClientConfig::getBufferLimit()-MEM_CHUNK_SIZE;
+ if(b!=NULL && AllocatedBuffer::s_allocatedMem < safeSize){
+ AllocatedBuffer::s_isBufferLimitReached=false;
+ //signal any waiting threads
+ AllocatedBuffer::s_memCV.notify_one();
+ }
+}
+
+
+AllocatedBuffer::AllocatedBuffer(size_t l){
+ m_pBuffer=NULL;
+ m_pBuffer=Utils::allocateBuffer(l);
+ m_bufSize=m_pBuffer!=NULL?l:0;
+}
+
+AllocatedBuffer::~AllocatedBuffer(){
+ Utils::freeBuffer(m_pBuffer, m_bufSize);
+ m_pBuffer=NULL;
+ m_bufSize=0;
+}
+
+} // namespace
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/clientlib/utils.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/clientlib/utils.hpp b/contrib/native/client/src/clientlib/utils.hpp
index 9def9b4..0f26ad6 100644
--- a/contrib/native/client/src/clientlib/utils.hpp
+++ b/contrib/native/client/src/clientlib/utils.hpp
@@ -23,24 +23,37 @@
#include <ostream>
#include <fstream>
#include <string>
-#include <stdlib.h>
+#include <boost/thread.hpp>
#include "drill/common.hpp"
+#include "drill/drillClient.hpp"
namespace Drill{
-class Utils{
+// Wrapper Class to keep track of allocated memory
+class AllocatedBuffer{
public:
+ AllocatedBuffer(size_t l);
+ ~AllocatedBuffer();
- //allocate memory for Record Batches
- static ByteBuf_t allocateBuffer(size_t len){
- //http://stackoverflow.com/questions/2688466/why-mallocmemset-is-slower-than-calloc
- ByteBuf_t b = (ByteBuf_t)calloc(len, sizeof(Byte_t)); return b;
- }
- static void freeBuffer(ByteBuf_t b){ free(b); }
+ ByteBuf_t m_pBuffer;
+ size_t m_bufSize;
+
+ // keep track of allocated memory. The client lib blocks
+ // if we have allocated up to a limit (defined in drillClientConfig).
+ static boost::mutex s_memCVMutex;
+ static boost::condition_variable s_memCV;
+ static size_t s_allocatedMem;
+ static bool s_isBufferLimitReached;
-}; // Utils
+};
+class Utils{
+ public:
+ //allocate memory for Record Batches
+ static ByteBuf_t allocateBuffer(size_t len);
+ static void freeBuffer(ByteBuf_t b, size_t len);
+}; // Utils
} // namespace Drill
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/include/drill/common.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/include/drill/common.hpp b/contrib/native/client/src/include/drill/common.hpp
index 2113ce5..59734dc 100644
--- a/contrib/native/client/src/include/drill/common.hpp
+++ b/contrib/native/client/src/include/drill/common.hpp
@@ -33,6 +33,9 @@
#define MAX_CONNECT_STR 4096
#define MAX_SOCK_RD_BUFSIZE 1024
+#define MEM_CHUNK_SIZE 64*1024; // 64K
+#define MAX_MEM_ALLOC_SIZE 256*1024*1024; // 256 MB
+
#ifdef _DEBUG
#define EXTRA_DEBUGGING
#define CODER_DEBUGGING
@@ -48,6 +51,9 @@ typedef Byte_t * ByteBuf_t;
class FieldMetadata;
typedef boost::shared_ptr< std::vector<Drill::FieldMetadata*> > FieldDefPtr;
+class AllocatedBuffer;
+typedef AllocatedBuffer* AllocatedBufferPtr;
+
typedef enum{
QRY_SUCCESS=0,
QRY_FAILURE=1,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a7feb9d/contrib/native/client/src/include/drill/recordBatch.hpp
----------------------------------------------------------------------
diff --git a/contrib/native/client/src/include/drill/recordBatch.hpp b/contrib/native/client/src/include/drill/recordBatch.hpp
index e9298bf..9a3df2b 100644
--- a/contrib/native/client/src/include/drill/recordBatch.hpp
+++ b/contrib/native/client/src/include/drill/recordBatch.hpp
@@ -836,10 +836,10 @@ class ValueVectorFactory{
class DECLSPEC_DRILL_CLIENT RecordBatch{
public:
- //m_allocatedBuffer is the memory block allocated to hold the incoming RPC message. Record BAtches operate on
- //slices of the allcoated buffer. The first slice (the first Field Batch), begins at m_buffer. Data in the
+ //m_allocatedBuffer is the memory block allocated to hold the incoming RPC message. Record Batches operate on
+ //slices of the allocated buffer. The first slice (the first Field Batch), begins at m_buffer. Data in the
//allocated buffer before m_buffer is mostly the RPC header, and the QueryResult object.
- RecordBatch(exec::shared::QueryResult* pResult, ByteBuf_t r, ByteBuf_t b)
+ RecordBatch(exec::shared::QueryResult* pResult, AllocatedBufferPtr r, ByteBuf_t b)
:m_fieldDefs(new(std::vector<Drill::FieldMetadata*>)){
m_pQueryResult=pResult;
m_pRecordBatchDef=&pResult->def();
@@ -892,7 +892,7 @@ class DECLSPEC_DRILL_CLIENT RecordBatch{
private:
const exec::shared::QueryResult* m_pQueryResult;
const exec::shared::RecordBatchDef* m_pRecordBatchDef;
- ByteBuf_t m_allocatedBuffer;
+ AllocatedBufferPtr m_allocatedBuffer;
ByteBuf_t m_buffer;
//build the current schema out of the field metadata
FieldDefPtr m_fieldDefs;
[15/47] git commit: DRILL-982: Return nulls for non-existent columns
in parquet reader.
Posted by ja...@apache.org.
DRILL-982: Return nulls for non-existent columns in parquet reader.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/fb93576a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/fb93576a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/fb93576a
Branch: refs/heads/master
Commit: fb93576a34367f38b9d040b36262bd20d88fbd35
Parents: c3e97fd
Author: Jason Altekruse <al...@gmail.com>
Authored: Fri Aug 15 09:31:19 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 12:09:34 2014 -0700
----------------------------------------------------------------------
.../columnreaders/ParquetRecordReader.java | 69 +++++++++++++++-----
.../store/parquet/ParquetRecordReaderTest.java | 24 +++++++
2 files changed, 78 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fb93576a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
index 4e9ac81..34e7aea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.ops.FragmentContext;
@@ -35,6 +36,7 @@ import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.NullableBitVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.hadoop.fs.FileSystem;
@@ -85,6 +87,18 @@ public class ParquetRecordReader implements RecordReader {
private List<SchemaPath> columns;
private FragmentContext fragmentContext;
private OperatorContext operatorContext;
+ // This is a parallel list to the columns list above, it is used to determine the subset of the project
+ // pushdown columns that do not appear in this file
+ private boolean[] columnsFound;
+ // For columns not found in the file, we need to return a schema element with the correct number of values
+ // at that position in the schema. Currently this requires a vector be present. Here is a list of all of these vectors
+ // that need only have their value count set at the end of each call to next(), as the values default to null.
+ private List<NullableBitVector> nullFilledVectors;
+ // Keeps track of the number of records returned in the case where only columns outside of the file were selected.
+ // No actual data needs to be read out of the file, we only need to return batches until we have 'read' the number of
+ // records specified in the row group metadata
+ long mockRecordsRead;
+
private final CodecFactoryExposer codecFactoryExposer;
int rowGroupIndex;
@@ -110,8 +124,10 @@ public class ParquetRecordReader implements RecordReader {
this.batchSize = batchSize;
this.footer = footer;
this.columns = columns;
- this.fragmentContext=fragmentContext;
-
+ if (this.columns != null) {
+ columnsFound = new boolean[this.columns.size()];
+ nullFilledVectors = new ArrayList();
+ }
}
public CodecFactoryExposer getCodecFactoryExposer() {
@@ -169,10 +185,13 @@ public class ParquetRecordReader implements RecordReader {
// for now it makes the existing tests pass, simply selecting
// all available data if no columns are provided
if (this.columns != null){
+ int i = 0;
for (SchemaPath expr : this.columns){
if ( field.matches(expr)){
+ columnsFound[i] = true;
return true;
}
+ i++;
}
return false;
}
@@ -189,6 +208,7 @@ public class ParquetRecordReader implements RecordReader {
ColumnDescriptor column;
ColumnChunkMetaData columnChunkMetaData;
int columnsToScan = 0;
+ mockRecordsRead = 0;
MaterializedField field;
ParquetMetadataConverter metaConverter = new ParquetMetadataConverter();
@@ -231,11 +251,7 @@ public class ParquetRecordReader implements RecordReader {
}
rowGroupOffset = footer.getBlocks().get(rowGroupIndex).getColumns().get(0).getFirstDataPageOffset();
- // none of the columns in the parquet file matched the request columns from the query
- if (columnsToScan == 0){
- throw new ExecutionSetupException("Error reading from parquet file. No columns requested were found in the file.");
- }
- if (allFieldsFixedLength) {
+ if (columnsToScan != 0 && allFieldsFixedLength) {
recordsPerBatch = (int) Math.min(Math.min(batchSize / bitWidthAllFixedFields,
footer.getBlocks().get(0).getColumns().get(0).getValueCount()), 65535);
}
@@ -245,16 +261,14 @@ public class ParquetRecordReader implements RecordReader {
try {
ValueVector v;
- ConvertedType convertedType;
SchemaElement schemaElement;
ArrayList<VarLengthColumn> varLengthColumns = new ArrayList<>();
// initialize all of the column read status objects
- boolean fieldFixedLength = false;
+ boolean fieldFixedLength;
for (int i = 0; i < columns.size(); ++i) {
column = columns.get(i);
columnChunkMetaData = footer.getBlocks().get(rowGroupIndex).getColumns().get(i);
schemaElement = schemaElements.get(column.getPath()[0]);
- convertedType = schemaElement.getConverted_type();
MajorType type = ParquetToDrillTypeConverter.toMajorType(column.getType(), schemaElement.getType_length(), getDataMode(column), schemaElement);
field = MaterializedField.create(toFieldName(column.getPath()), type);
// the field was not requested to be read
@@ -279,6 +293,16 @@ public class ParquetRecordReader implements RecordReader {
}
}
varLengthReader = new VarLenBinaryReader(this, varLengthColumns);
+
+ if (this.columns != null) {
+ for (int i = 0; i < columnsFound.length; i++) {
+ if ( ! columnsFound[i]) {
+ nullFilledVectors.add((NullableBitVector)output.addField(MaterializedField.create(this.columns.get(i), Types.optional(TypeProtos.MinorType.BIT)),
+ (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(TypeProtos.MinorType.BIT, DataMode.OPTIONAL)));
+
+ }
+ }
+ }
} catch (SchemaChangeException e) {
throw new ExecutionSetupException(e);
} catch (Exception e) {
@@ -333,11 +357,18 @@ public class ParquetRecordReader implements RecordReader {
firstColumnStatus = null;
}
}
- // TODO - replace this with new functionality of returning batches even if no columns are selected
- // the query 'select 5 from parquetfile' should return the number of records that the parquet file contains
- // we don't need to read any of the data, we just need to fill batches with a record count and a useless vector with
- // the right number of values
- if (firstColumnStatus == null) throw new DrillRuntimeException("Unexpected error reading parquet file, not reading any columns");
+ // No columns found in the file were selected, simply return a full batch of null records for each column requested
+ if (firstColumnStatus == null) {
+ if (mockRecordsRead == footer.getBlocks().get(rowGroupIndex).getRowCount()) {
+ return 0;
+ }
+ recordsToRead = Math.min(DEFAULT_RECORDS_TO_READ_IF_NOT_FIXED_WIDTH, footer.getBlocks().get(rowGroupIndex).getRowCount() - mockRecordsRead);
+ for (ValueVector vv : nullFilledVectors ) {
+ vv.getMutator().setValueCount( (int) recordsToRead);
+ }
+ mockRecordsRead += recordsToRead;
+ return (int) recordsToRead;
+ }
if (allFieldsFixedLength) {
recordsToRead = Math.min(recordsPerBatch, firstColumnStatus.columnChunkMetaData.getValueCount() - firstColumnStatus.totalValuesRead);
@@ -353,6 +384,14 @@ public class ParquetRecordReader implements RecordReader {
readAllFixedFields(fixedRecordsToRead);
}
+ // if we have requested columns that were not found in the file fill their vectors with null
+ // (by simply setting the value counts inside of them, as they start null filled)
+ if (nullFilledVectors != null) {
+ for (ValueVector vv : nullFilledVectors ) {
+ vv.getMutator().setValueCount(firstColumnStatus.getRecordsReadInCurrentPass());
+ }
+ }
+
return firstColumnStatus.getRecordsReadInCurrentPass();
} catch (IOException e) {
throw new DrillRuntimeException(e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fb93576a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index 2193233..cbae6ba 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -37,6 +37,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -58,8 +59,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.BeforeClass;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestRule;
import parquet.bytes.BytesInput;
import parquet.column.page.Page;
import parquet.column.page.PageReadStore;
@@ -139,6 +142,27 @@ public class ParquetRecordReaderTest extends BaseTestQuery{
}
@Test
+ public void testNonExistentColumn() throws Exception {
+ testFull(QueryType.SQL, "select non_existent_column from cp.`tpch/nation.parquet`", "", 1, 1, 150000, false);
+ }
+
+
+ @Test
+ public void testNonExistentColumnLargeFile() throws Exception {
+ testFull(QueryType.SQL, "select non_existent_column, non_existent_col_2 from dfs.`/tmp/customer.dict.parquet`", "", 1, 1, 150000, false);
+ }
+
+ @Test
+ public void testNonExistentColumnsSomePresentColumnsLargeFile() throws Exception {
+ testFull(QueryType.SQL, "select cust_key, address, non_existent_column, non_existent_col_2 from dfs.`/tmp/customer.dict.parquet`", "", 1, 1, 150000, false);
+ }
+
+ @Test
+ public void testTPCHPerformace_SF1() throws Exception {
+ testFull(QueryType.SQL, "select * from dfs.`/tmp/orders_part-m-00001.parquet`", "", 1, 1, 150000, false);
+ }
+
+ @Test
public void testLocalDistributed() throws Exception {
String planName = "/parquet/parquet_scan_union_screen_physical.json";
testParquetFullEngineLocalTextDistributed(planName, fileName, 1, numberRowGroups, recordsPerRowGroup);
[17/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
new file mode 100644
index 0000000..8e752eb
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
@@ -0,0 +1,692 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import io.netty.buffer.DrillBuf;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.util.CoreDecimalUtility;
+
+public class DecimalUtility extends CoreDecimalUtility{
+
+ public final static int MAX_DIGITS = 9;
+ public final static int DIGITS_BASE = 1000000000;
+ public final static int DIGITS_MAX = 999999999;
+ public final static int integerSize = (Integer.SIZE/8);
+
+ public final static String[] decimalToString = {"",
+ "0",
+ "00",
+ "000",
+ "0000",
+ "00000",
+ "000000",
+ "0000000",
+ "00000000",
+ "000000000"};
+
+ public final static long[] scale_long_constants = {
+ 1,
+ 10,
+ 100,
+ 1000,
+ 10000,
+ 100000,
+ 1000000,
+ 10000000,
+ 100000000,
+ 1000000000,
+ 10000000000l,
+ 100000000000l,
+ 1000000000000l,
+ 10000000000000l,
+ 100000000000000l,
+ 1000000000000000l,
+ 10000000000000000l,
+ 100000000000000000l,
+ 1000000000000000000l};
+
+ /*
+ * Simple function that returns the static precomputed
+ * power of ten, instead of using Math.pow
+ */
+ public static long getPowerOfTen(int power) {
+ assert power >= 0 && power < scale_long_constants.length;
+ return scale_long_constants[(power)];
+ }
+
+ /*
+ * Math.pow returns a double and while multiplying with large digits
+ * in the decimal data type we encounter noise. So instead of multiplying
+ * with Math.pow we use the static constants to perform the multiplication
+ */
+ public static long adjustScaleMultiply(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input * scale_long_constants[index];
+ } else {
+ return input / scale_long_constants[index];
+ }
+ }
+
+ public static long adjustScaleDivide(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input / scale_long_constants[index];
+ } else {
+ return input * scale_long_constants[index];
+ }
+ }
+
+ /* Given the number of actual digits this function returns the
+ * number of indexes it will occupy in the array of integers
+ * which are stored in base 1 billion
+ */
+ public static int roundUp(int ndigits) {
+ return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+ }
+
+ /* Returns a string representation of the given integer
+ * If the length of the given integer is less than the
+ * passed length, this function will prepend zeroes to the string
+ */
+ public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+ String value = ((Integer) number).toString();
+ int length = value.length();
+
+ StringBuilder str = new StringBuilder();
+ str.append(decimalToString[desiredLength - length]);
+ str.append(value);
+
+ return str;
+ }
+
+ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+ String value = ((Long) number).toString();
+ int length = value.length();
+
+ StringBuilder str = new StringBuilder();
+
+ // Desired length can be > MAX_DIGITS
+ int zeroesLength = desiredLength - length;
+ while (zeroesLength > MAX_DIGITS) {
+ str.append(decimalToString[MAX_DIGITS]);
+ zeroesLength -= MAX_DIGITS;
+ }
+ str.append(decimalToString[zeroesLength]);
+ str.append(value);
+
+ return str;
+ }
+
+ public static BigDecimal getBigDecimalFromIntermediate(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+ // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+ return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
+ }
+
+ public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+ // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+ return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
+ }
+
+ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) {
+ byte[] value = new byte[length];
+ bytebuf.getBytes(start, value, 0, length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ /* Create a BigDecimal object using the data in the DrillBuf.
+ * This function assumes that data is provided in a non-dense format
+ * It works on both sparse and intermediate representations.
+ */
+ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) {
+
+ // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+ int actualDigits;
+
+ // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
+ BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+
+ BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+
+ for (int i = 1; i < nDecimalDigits; i++) {
+
+ BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize)));
+ decimalDigits = decimalDigits.multiply(base);
+ decimalDigits = decimalDigits.add(temp);
+ }
+
+ // Truncate any additional padding we might have added
+ if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+ BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+ decimalDigits = decimalDigits.divide(truncate);
+ }
+
+ // set the sign
+ if ((data.getInt(startIndex) & 0x80000000) != 0) {
+ decimalDigits = decimalDigits.negate();
+ }
+
+ BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+
+ return decimal;
+ }
+
+ /* This function returns a BigDecimal object from the dense decimal representation.
+ * First step is to convert the dense representation into an intermediate representation
+ * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
+ */
+ public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+
+ /* This method converts the dense representation to
+ * an intermediate representation. The intermediate
+ * representation has one more integer than the dense
+ * representation.
+ */
+ byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)];
+
+ // Start storing from the least significant byte of the first integer
+ int intermediateIndex = 3;
+
+ int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+ int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+ int maskIndex;
+ int shiftOrder;
+ byte shiftBits;
+
+ // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+ if (maxPrecision == 38) {
+ maskIndex = 0;
+ shiftOrder = 6;
+ shiftBits = 0x00;
+ intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+ } else if (maxPrecision == 28) {
+ maskIndex = 1;
+ shiftOrder = 4;
+ shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+ intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+ } else {
+ throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+ }
+
+ int inputIndex = 1;
+ boolean sign = false;
+
+ if ((data.getByte(startIndex) & 0x80) != 0) {
+ sign = true;
+ }
+
+ while (inputIndex < width) {
+
+ intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+
+ shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+
+ inputIndex++;
+ intermediateIndex++;
+
+ if (((inputIndex - 1) % integerSize) == 0) {
+ shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+ maskIndex++;
+ shiftOrder -= 2;
+ }
+
+ }
+ /* copy the last byte */
+ intermediateBytes[intermediateIndex] = shiftBits;
+
+ if (sign == true) {
+ intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+ }
+ DrillBuf intermediate = data.getAllocator().buffer(intermediateBytes.length);
+ intermediate.setBytes(0, intermediateBytes);
+
+ BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
+ intermediate.release();
+ return ret;
+ }
+
+ /*
+ * Function converts the BigDecimal and stores it in out internal sparse representation
+ */
+ public static void getSparseFromBigDecimal(BigDecimal input, DrillBuf data, int startIndex, int scale, int precision, int nDecimalDigits) {
+
+ // Initialize the buffer
+ for (int i = 0; i < nDecimalDigits; i++) {
+ data.setInt(startIndex + (i * integerSize), 0);
+ }
+
+ boolean sign = false;
+
+ if (input.signum() == -1) {
+ // negative input
+ sign = true;
+ input = input.abs();
+ }
+
+ // Truncate the input as per the scale provided
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ // Separate out the integer part
+ BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+
+ int destIndex = nDecimalDigits - roundUp(scale) - 1;
+
+ // we use base 1 billion integer digits for out integernal representation
+ BigDecimal base = new BigDecimal(DIGITS_BASE);
+
+ while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+ // store the modulo as the integer value
+ data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue());
+ destIndex--;
+ // Divide by base 1 billion
+ integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+ }
+
+ /* Sparse representation contains padding of additional zeroes
+ * so each digit contains MAX_DIGITS for ease of arithmetic
+ */
+ int actualDigits;
+ if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+ // Pad additional zeroes
+ scale = scale + (MAX_DIGITS - actualDigits);
+ input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+ }
+
+ //separate out the fractional part
+ BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+
+ destIndex = nDecimalDigits - 1;
+
+ while (scale > 0) {
+ // Get next set of MAX_DIGITS (9) store it in the DrillBuf
+ fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+ BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+
+ data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue()));
+ destIndex--;
+
+ fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+ scale -= MAX_DIGITS;
+ }
+
+ // Set the negative sign
+ if (sign == true) {
+ data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
+ }
+
+ }
+
+
+ public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+ // Truncate or pad to set the input to the correct scale
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ return (input.unscaledValue().longValue());
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+
+ public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) {
+
+ int invert = 1;
+
+ /* If signs are different then simply look at the
+ * sign of the two inputs and determine which is greater
+ */
+ if (leftSign != rightSign) {
+
+ return((leftSign == true) ? -1 : 1);
+ } else if(leftSign == true) {
+ /* Both inputs are negative, at the end we will
+ * have to invert the comparison
+ */
+ invert = -1;
+ }
+
+ int cmp = 0;
+
+ for (int i = 0; i < width; i++) {
+ byte leftByte = left.getByte(leftStart + i);
+ byte rightByte = right.getByte(rightStart + i);
+ // Unsigned byte comparison
+ if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+ cmp = 1;
+ break;
+ } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+ cmp = -1;
+ break;
+ }
+ }
+ cmp *= invert; // invert the comparison if both were negative values
+
+ return cmp;
+ }
+
+ public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) {
+ int value = buffer.getInt(start + (index * 4));
+
+ if (index == 0) {
+ /* the first byte contains sign bit, return value without it */
+ value = (value & 0x7FFFFFFF);
+ }
+ return value;
+ }
+
+ public static void setInteger(DrillBuf buffer, int start, int index, int value) {
+ buffer.setInt(start + (index * 4), value);
+ }
+
+ public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+
+ int invert = 1;
+
+ if (absCompare == false) {
+ if (leftSign != rightSign) {
+ return (leftSign == true) ? -1 : 1;
+ }
+
+ // Both values are negative invert the outcome of the comparison
+ if (leftSign == true) {
+ invert = -1;
+ }
+ }
+
+ int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+ return cmp * invert;
+ }
+ public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+ /* compute the number of integer digits in each decimal */
+ int leftInt = leftPrecision - leftScale;
+ int rightInt = rightPrecision - rightScale;
+
+ /* compute the number of indexes required for storing integer digits */
+ int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt);
+ int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt);
+
+ /* compute number of indexes required for storing scale */
+ int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale);
+ int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale);
+
+ /* compute index of the most significant integer digits */
+ int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+ int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+ int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+
+ /* Discard the zeroes in the integer part */
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ break;
+ }
+
+ /* Digit in this location is zero, decrement the actual number
+ * of integer digits
+ */
+ leftIntRoundedUp--;
+ leftIndex1++;
+ }
+
+ /* If we reached the stop index then the number of integers is zero */
+ if (leftIndex1 == leftStopIndex) {
+ leftIntRoundedUp = 0;
+ }
+
+ while (rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ break;
+ }
+
+ /* Digit in this location is zero, decrement the actual number
+ * of integer digits
+ */
+ rightIntRoundedUp--;
+ rightIndex1++;
+ }
+
+ if (rightIndex1 == rightStopIndex) {
+ rightIntRoundedUp = 0;
+ }
+
+ /* We have the accurate number of non-zero integer digits,
+ * if the number of integer digits are different then we can determine
+ * which decimal is larger and needn't go down to comparing individual values
+ */
+ if (leftIntRoundedUp > rightIntRoundedUp) {
+ return 1;
+ }
+ else if (rightIntRoundedUp > leftIntRoundedUp) {
+ return -1;
+ }
+
+ /* The number of integer digits are the same, set the each index
+ * to the first non-zero integer and compare each digit
+ */
+ leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
+ }
+
+ leftIndex1++;
+ rightIndex1++;
+ }
+
+ /* The integer part of both the decimal's are equal, now compare
+ * each individual fractional part. Set the index to be at the
+ * beginning of the fractional part
+ */
+ leftIndex1 = leftStopIndex;
+ rightIndex1 = rightStopIndex;
+
+ /* Stop indexes will be the end of the array */
+ leftStopIndex = nDecimalDigits;
+ rightStopIndex = nDecimalDigits;
+
+ /* compare the two fractional parts of the decimal */
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
+ }
+
+ leftIndex1++;
+ rightIndex1++;
+ }
+
+ /* Till now the fractional part of the decimals are equal, check
+ * if one of the decimal has fractional part that is remaining
+ * and is non-zero
+ */
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ return 1;
+ }
+ leftIndex1++;
+ }
+
+ while(rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ return -1;
+ }
+ rightIndex1++;
+ }
+
+ /* Both decimal values are equal */
+ return 0;
+ }
+
+ public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+ byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
+ int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale);
+ int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale);
+
+ if (desiredScale < currentScale) {
+
+ boolean roundUp = false;
+
+ //Extract the first digit to be truncated to check if we need to round up
+ int truncatedScaleIndex = desiredScale + 1;
+ if (truncatedScaleIndex <= currentScale) {
+ int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
+ extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex);
+ int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
+ int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (temp != 0) {
+ extractDigit = extractDigit / (int) (Math.pow(10, temp));
+ }
+ if ((extractDigit % 10) > 4) {
+ roundUp = true;
+ }
+ }
+
+ // Get the source index beyond which we will truncate
+ int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
+ int srcIndex = srcIntIndex + newScaleRoundedUp;
+
+ // Truncate the remaining fractional part, move the integer part
+ int destIndex = nDecimalDigits - 1;
+ if (srcIndex != destIndex) {
+ while (srcIndex >= 0) {
+ setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
+ }
+
+ // Set the remaining portion of the decimal to be zeroes
+ while (destIndex >= 0) {
+ setInteger(result, start, destIndex--, 0);
+ }
+ srcIndex = nDecimalDigits - 1;
+ }
+
+ // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
+ int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ truncateFactor = (int) Math.pow(10, truncateFactor);
+ int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
+ fractionalDigits /= truncateFactor;
+ setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
+ }
+
+ // Finally round up the digit if needed
+ if (roundUp == true) {
+ srcIndex = nDecimalDigits - 1;
+ int carry;
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ carry = truncateFactor;
+ } else {
+ carry = 1;
+ }
+
+ while (srcIndex >= 0) {
+ int value = getIntegerFromSparseBuffer(result, start, srcIndex);
+ value += carry;
+
+ if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
+ carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
+ } else {
+ setInteger(result, start, srcIndex--, value);
+ carry = 0;
+ break;
+ }
+ }
+ }
+ } else if (desiredScale > currentScale) {
+ // Add fractional digits to the decimal
+
+ // Check if we need to shift the decimal digits to the left
+ if (newScaleRoundedUp > origScaleRoundedUp) {
+ int srcIndex = 0;
+ int destIndex = newScaleRoundedUp - origScaleRoundedUp;
+
+ // Check while extending scale, we are not overwriting integer part
+ while (srcIndex < destIndex) {
+ if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
+ }
+ }
+
+ srcIndex = 0;
+ while (destIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
+ }
+
+ // Clear the remaining part
+ while (srcIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, 0);
+ }
+ }
+ }
+ }
+
+ public static int getFirstFractionalDigit(int decimal, int scale) {
+ if (scale == 0) {
+ return 0;
+ }
+ int temp = (int) adjustScaleDivide(decimal, scale - 1);
+ return Math.abs(temp % 10);
+ }
+
+ public static int getFirstFractionalDigit(long decimal, int scale) {
+ if (scale == 0) {
+ return 0;
+ }
+ long temp = adjustScaleDivide(decimal, scale - 1);
+ return (int) (Math.abs(temp % 10));
+ }
+
+ public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) {
+ if (scale == 0) {
+ return 0;
+ }
+
+ int index = nDecimalDigits - roundUp(scale);
+ return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
index 7d0fbc7..b711c66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -17,9 +17,10 @@
*/
package org.apache.drill.exec.vector;
-import java.util.Iterator;
+import io.netty.buffer.DrillBuf;
+import io.netty.buffer.DrillBuf;
-import io.netty.buffer.ByteBuf;
+import java.util.Iterator;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -30,7 +31,7 @@ import com.google.common.collect.Iterators;
public abstract class BaseDataValueVector extends BaseValueVector{
- protected ByteBuf data = DeadBuf.DEAD_BUFFER;
+ protected DrillBuf data;
protected int valueCount;
protected int currentValueCount;
@@ -40,13 +41,13 @@ public abstract class BaseDataValueVector extends BaseValueVector{
}
/**
- * Release the underlying ByteBuf and reset the ValueVector
+ * Release the underlying DrillBuf and reset the ValueVector
*/
@Override
public void clear() {
- if (data != DeadBuf.DEAD_BUFFER) {
+ if (data != null) {
data.release();
- data = DeadBuf.DEAD_BUFFER;
+ data = data.getAllocator().getEmpty();
valueCount = 0;
}
}
@@ -61,12 +62,12 @@ public abstract class BaseDataValueVector extends BaseValueVector{
@Override
- public ByteBuf[] getBuffers(){
- ByteBuf[] out;
+ public DrillBuf[] getBuffers(){
+ DrillBuf[] out;
if(valueCount == 0){
- out = new ByteBuf[0];
+ out = new DrillBuf[0];
}else{
- out = new ByteBuf[]{data};
+ out = new DrillBuf[]{data};
data.readerIndex(0);
data.retain();
}
@@ -82,10 +83,14 @@ public abstract class BaseDataValueVector extends BaseValueVector{
@Override
public abstract SerializedField getMetadata();
- public ByteBuf getData(){
+ public DrillBuf getData(){
return data;
}
+ public long getDataAddr(){
+ return data.memoryAddress();
+ }
+
@Override
public Iterator<ValueVector> iterator() {
return Iterators.emptyIterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index e310b81..cf62d54 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -63,7 +63,7 @@ public abstract class BaseValueVector implements ValueVector{
abstract public ByteBuf getData();
- abstract class BaseAccessor implements ValueVector.Accessor{
+ abstract static class BaseAccessor implements ValueVector.Accessor{
public abstract int getValueCount();
public void reset(){}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 01fcfe0..0076a1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -17,8 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.AccountingByteBuf;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.holders.BitHolder;
@@ -113,11 +112,11 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
}
@Override
- public int load(int valueCount, ByteBuf buf) {
+ public int load(int valueCount, DrillBuf buf) {
clear();
this.valueCount = valueCount;
int len = getSizeFromCount(valueCount);
- data = buf.slice(0, len);
+ data = (DrillBuf) buf.slice(0, len);
data.retain();
return len;
}
@@ -136,7 +135,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
+ public void load(SerializedField metadata, DrillBuf buffer) {
assert this.field.matches(metadata);
int loaded = load(metadata.getValueCount(), buffer);
assert metadata.getBufferLength() == loaded;
@@ -181,7 +180,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
int offset = startIndex % 8;
if (offset == 0) {
// slice
- target.data = this.data.slice(firstByte, byteSize);
+ target.data = (DrillBuf) this.data.slice(firstByte, byteSize);
target.data.retain();
} else {
// Copy data
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
index c5e1ae7..b546b9e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -17,26 +17,26 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface FixedWidthVector extends ValueVector{
-
+
/**
* Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
*
* @param valueCount Number of values in the vector.
*/
public void allocateNew(int valueCount);
-
+
/**
* Load the records in the provided buffer based on the given number of values.
* @param valueCount Number of values the buffer contains.
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int valueCount, ByteBuf buf);
-
-
+ public int load(int valueCount, DrillBuf buf);
+
+
public abstract Mutator getMutator();
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index 650f5ae..2c215ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.vector;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.holders.ObjectHolder;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -164,12 +166,12 @@ public class ObjectVector extends BaseValueVector{
}
@Override
- public ByteBuf[] getBuffers() {
+ public DrillBuf[] getBuffers() {
throw new UnsupportedOperationException("ObjectVector does not support this");
}
@Override
- public void load(UserBitShared.SerializedField metadata, ByteBuf buffer) {
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
throw new UnsupportedOperationException("ObjectVector does not support this");
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
index 6d03541..a28caf1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface RepeatedFixedWidthVector extends ValueVector{
/**
@@ -27,7 +27,7 @@ public interface RepeatedFixedWidthVector extends ValueVector{
* @param childValueCount Number of supported values in the vector.
*/
public void allocateNew(int parentValueCount, int childValueCount);
-
+
/**
* Load the records in the provided buffer based on the given number of values.
* @param parentValueCount Number of separate repeating groupings.
@@ -35,7 +35,7 @@ public interface RepeatedFixedWidthVector extends ValueVector{
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int parentValueCount, int childValueCount, ByteBuf buf);
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf);
public abstract RepeatedAccessor getAccessor();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
index a2c884e..2e7689c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface RepeatedVariableWidthVector extends ValueVector{
/**
@@ -45,5 +45,5 @@ public interface RepeatedVariableWidthVector extends ValueVector{
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf);
+ public int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
index 6968c21..31afb6f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -17,20 +17,18 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.SwappedByteBuf;
-import io.netty.buffer.Unpooled;
-import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.buffer.DrillBuf;
import java.math.BigDecimal;
-import java.nio.ByteOrder;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.Decimal18Holder;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal9Holder;
import org.apache.drill.exec.expr.holders.IntervalDayHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.util.DecimalUtility;
import com.google.common.base.Charsets;
@@ -38,13 +36,24 @@ import com.google.common.base.Charsets;
public class ValueHolderHelper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
- public static VarCharHolder getVarCharHolder(String s){
+ public static VarCharHolder getVarCharHolder(DrillBuf buf, String s){
VarCharHolder vch = new VarCharHolder();
byte[] b = s.getBytes(Charsets.UTF_8);
vch.start = 0;
vch.end = b.length;
- vch.buffer = UnpooledByteBufAllocator.DEFAULT.buffer(b.length).order(ByteOrder.LITTLE_ENDIAN); // use the length of input string to allocate buffer.
+ vch.buffer = buf.reallocIfNeeded(b.length);
+ vch.buffer.setBytes(0, b);
+ return vch;
+ }
+
+ public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
+ VarCharHolder vch = new VarCharHolder();
+
+ byte[] b = s.getBytes(Charsets.UTF_8);
+ vch.start = 0;
+ vch.end = b.length;
+ vch.buffer = a.buffer(b.length); //
vch.buffer.setBytes(0, b);
return vch;
}
@@ -53,7 +62,7 @@ public class ValueHolderHelper {
IntervalDayHolder dch = new IntervalDayHolder();
dch.days = days;
- dch.milliSeconds = millis;
+ dch.milliseconds = millis;
return dch;
}
@@ -77,7 +86,7 @@ public class ValueHolderHelper {
return dch;
}
- public static Decimal28SparseHolder getDecimal28Holder(String decimal) {
+ public static Decimal28SparseHolder getDecimal28Holder(DrillBuf buf, String decimal) {
Decimal28SparseHolder dch = new Decimal28SparseHolder();
@@ -85,18 +94,15 @@ public class ValueHolderHelper {
dch.scale = bigDecimal.scale();
dch.precision = bigDecimal.precision();
- dch.setSign(bigDecimal.signum() == -1);
+ Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
-
- dch.buffer = Unpooled.wrappedBuffer(new byte[5 * DecimalUtility.integerSize]);
- dch.buffer = new SwappedByteBuf(dch.buffer);
+ dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.integerSize);
DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
return dch;
}
- public static Decimal38SparseHolder getDecimal38Holder(String decimal) {
-
+ public static Decimal38SparseHolder getDecimal38Holder(DrillBuf buf, String decimal) {
Decimal38SparseHolder dch = new Decimal38SparseHolder();
@@ -104,12 +110,9 @@ public class ValueHolderHelper {
dch.scale = bigDecimal.scale();
dch.precision = bigDecimal.precision();
- dch.setSign(bigDecimal.signum() == -1);
+ Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
-
-
- dch.buffer = Unpooled.wrappedBuffer(new byte[dch.maxPrecision * DecimalUtility.integerSize]);
- dch.buffer = new SwappedByteBuf(dch.buffer);
+ dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.integerSize);
DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
return dch;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 8b871fc..f7f010a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.io.Closeable;
@@ -56,7 +56,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
public void close();
/**
- * Release the underlying ByteBuf and reset the ValueVector to empty.
+ * Release the underlying DrillBuf and reset the ValueVector to empty.
*/
public void clear();
@@ -99,9 +99,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
* external classes shouldn't hold a reference to it (unless they change it).
*
- * @return The underlying ByteBuf.
+ * @return The underlying DrillBuf.
*/
- public abstract ByteBuf[] getBuffers();
+ public abstract DrillBuf[] getBuffers();
/**
* Load the data provided in the buffer. Typically used when deserializing from the wire.
@@ -111,7 +111,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param buffer
* The buffer that contains the ValueVector.
*/
- public void load(SerializedField metadata, ByteBuf buffer);
+ public void load(SerializedField metadata, DrillBuf buffer);
/**
* Get the metadata for this field. Used in serialization
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
index 5c1d3ab..ff3ee63 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface VariableWidthVector extends ValueVector{
@@ -28,15 +28,13 @@ public interface VariableWidthVector extends ValueVector{
* @param valueCount Number of values in the vector.
*/
public void allocateNew(int totalBytes, int valueCount);
-
+
/**
* Provide the maximum amount of variable width bytes that can be stored int his vector.
* @return
*/
public int getByteCapacity();
- public int getCurrentSizeInBytes();
-
/**
* Load the records in the provided buffer based on the given number of values.
* @param dataBytes The number of bytes associated with the data array.
@@ -44,8 +42,8 @@ public interface VariableWidthVector extends ValueVector{
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int dataBytes, int valueCount, ByteBuf buf);
-
+ public int load(int dataBytes, int valueCount, DrillBuf buf);
+
public abstract VariableWidthMutator getMutator();
public abstract VariableWidthAccessor getAccessor();
@@ -54,6 +52,8 @@ public interface VariableWidthVector extends ValueVector{
public int getValueLength(int index);
}
+ public int getCurrentSizeInBytes();
+
public interface VariableWidthMutator extends Mutator {
public boolean setValueLengthSafe(int index, int length);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
index e946afe..619dd95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.ByteBuf;
public class VectorTrimmer {
@@ -25,7 +25,7 @@ public class VectorTrimmer {
public static void trim(ByteBuf data, int idx) {
data.writerIndex(idx);
- if (data instanceof AccountingByteBuf) {
+ if (data instanceof DrillBuf) {
// data.capacity(idx);
data.writerIndex(idx);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 480b863..c91c397 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector.complex;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.HashMap;
import java.util.Iterator;
@@ -276,18 +276,18 @@ public class MapVector extends AbstractContainerVector {
}
@Override
- public ByteBuf[] getBuffers() {
- List<ByteBuf> bufs = Lists.newArrayList();
+ public DrillBuf[] getBuffers() {
+ List<DrillBuf> bufs = Lists.newArrayList();
for(ValueVector v : vectors.values()){
- for(ByteBuf b : v.getBuffers()){
+ for(DrillBuf b : v.getBuffers()){
bufs.add(b);
}
}
- return bufs.toArray(new ByteBuf[bufs.size()]);
+ return bufs.toArray(new DrillBuf[bufs.size()]);
}
@Override
- public void load(SerializedField metadata, ByteBuf buf) {
+ public void load(SerializedField metadata, DrillBuf buf) {
List<SerializedField> fields = metadata.getChildList();
valueCount = metadata.getValueCount();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 57c47d4..00716cf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector.complex;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.Collections;
import java.util.Iterator;
@@ -354,7 +354,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
}
@Override
- public ByteBuf[] getBuffers() {
+ public DrillBuf[] getBuffers() {
return ArrayUtils.addAll(offsets.getBuffers(), vector.getBuffers());
}
@@ -364,7 +364,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
}
@Override
- public void load(SerializedField metadata, ByteBuf buf) {
+ public void load(SerializedField metadata, DrillBuf buf) {
SerializedField childField = metadata.getChildList().get(0);
int bufOffset = offsets.load(metadata.getValueCount()+1, buf);
@@ -429,7 +429,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
}
@Override
- public int load(int parentValueCount, int childValueCount, ByteBuf buf) {
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index c67c047..f7baaa9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector.complex;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.Iterator;
import java.util.List;
@@ -307,20 +307,20 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
@Override
- public ByteBuf[] getBuffers() {
- List<ByteBuf> bufs = Lists.newArrayList(offsets.getBuffers());
+ public DrillBuf[] getBuffers() {
+ List<DrillBuf> bufs = Lists.newArrayList(offsets.getBuffers());
for(ValueVector v : vectors.values()){
- for(ByteBuf b : v.getBuffers()){
+ for(DrillBuf b : v.getBuffers()){
bufs.add(b);
}
}
- return bufs.toArray(new ByteBuf[bufs.size()]);
+ return bufs.toArray(new DrillBuf[bufs.size()]);
}
@Override
- public void load(SerializedField metadata, ByteBuf buf) {
+ public void load(SerializedField metadata, DrillBuf buf) {
List<SerializedField> fields = metadata.getChildList();
int bufOffset = offsets.load(metadata.getValueCount()+1, buf);
@@ -513,7 +513,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
@Override
- public int load(int parentValueCount, int childValueCount, ByteBuf buf) {
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index d365c6f..59035cb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -17,9 +17,7 @@
*/
package org.apache.drill.exec.vector.complex.fn;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.io.Reader;
@@ -33,11 +31,10 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.core.JsonToken;
import com.google.common.base.Charsets;
public class JsonReader {
@@ -48,10 +45,12 @@ public class JsonReader {
private final JsonFactory factory = new JsonFactory();
private JsonParser parser;
+ private DrillBuf workBuf;
- public JsonReader() throws JsonParseException, IOException {
+ public JsonReader(DrillBuf managedBuf) throws JsonParseException, IOException {
factory.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
factory.configure(Feature.ALLOW_COMMENTS, true);
+ this.workBuf = managedBuf;
}
public boolean write(Reader reader, ComplexWriter writer) throws JsonParseException, IOException {
@@ -134,9 +133,9 @@ public class JsonReader {
VarCharHolder vh = new VarCharHolder();
String value = parser.getText();
byte[] b = value.getBytes(Charsets.UTF_8);
- ByteBuf d = UnpooledByteBufAllocator.DEFAULT.buffer(b.length);
- d.setBytes(0, b);
- vh.buffer = d;
+ ensure(b.length);
+ workBuf.setBytes(0, b);
+ vh.buffer = workBuf;
vh.start = 0;
vh.end = b.length;
map.varChar(fieldName).write(vh);
@@ -152,6 +151,10 @@ public class JsonReader {
}
+ private void ensure(int length){
+ workBuf = workBuf.reallocIfNeeded(length);
+ }
+
private void writeData(ListWriter list) throws JsonParseException, IOException {
list.start();
outside: while(true){
@@ -197,9 +200,9 @@ public class JsonReader {
VarCharHolder vh = new VarCharHolder();
String value = parser.getText();
byte[] b = value.getBytes(Charsets.UTF_8);
- ByteBuf d = UnpooledByteBufAllocator.DEFAULT.buffer(b.length);
- d.setBytes(0, b);
- vh.buffer = d;
+ ensure(b.length);
+ workBuf.setBytes(0, b);
+ vh.buffer = workBuf;
vh.start = 0;
vh.end = b.length;
list.varChar().write(vh);
@@ -209,7 +212,7 @@ public class JsonReader {
}
}
list.end();
-
-
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
index cf885a4..54e063b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.vector.complex.fn;
+import io.netty.buffer.DrillBuf;
+
import java.io.IOException;
import java.io.Reader;
@@ -35,10 +37,10 @@ public class JsonReaderWithState {
private JsonRecordSplitter splitter;
private JsonReader jsonReader;
- public JsonReaderWithState(JsonRecordSplitter splitter) throws IOException{
+ public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace) throws IOException{
this.splitter = splitter;
reader = splitter.getNextReader();
- jsonReader = new JsonReader();
+ jsonReader = new JsonReader(workspace);
}
public WriteState write(ComplexWriter writer) throws JsonParseException, IOException {
@@ -50,7 +52,7 @@ public class JsonReaderWithState {
}
jsonReader.write(reader, writer);
-
+
if (!writer.ok()) {
reader.reset();
return WriteState.WRITE_FAILED;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
index 76d642c..c7527d5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.work.batch;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.util.List;
@@ -36,10 +37,6 @@ import org.apache.drill.exec.proto.BitData;
import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.record.RawFragmentBatch;
-import org.apache.drill.exec.rpc.RemoteConnection;
-import org.apache.drill.exec.rpc.ResponseSender;
-import org.apache.drill.exec.rpc.data.BitServerConnection;
-import org.apache.drill.exec.rpc.data.DataRpcConfig;
import org.apache.drill.exec.store.LocalSyncableFileSystem;
import org.apache.drill.exec.work.fragment.FragmentManager;
import org.apache.hadoop.conf.Configuration;
@@ -272,7 +269,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
Stopwatch watch = new Stopwatch();
watch.start();
BitData.FragmentRecordBatch header = BitData.FragmentRecordBatch.parseDelimitedFrom(stream);
- ByteBuf buf = allocator.buffer(bodyLength);
+ DrillBuf buf = allocator.buffer(bodyLength);
buf.writeBytes(stream, bodyLength);
batch = new RawFragmentBatch(null, header, buf, null);
buf.release();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java
new file mode 100644
index 0000000..4258e60
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec;
+
+
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Files;
+
+public class RunRootExec {
+ public static DrillConfig c = DrillConfig.create();
+
+ public static void main(String args[]) throws Exception {
+ String path = args[0];
+ int iterations = Integer.parseInt(args[1]);
+ Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet());
+ bit.run();
+ DrillbitContext bitContext = bit.getContext();
+ PhysicalPlanReader reader = bitContext.getPlanReader();
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(new File(path), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = bitContext.getFunctionImplementationRegistry();
+ FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), null, registry);
+ SimpleRootExec exec;
+ for (int i = 0; i < iterations; i ++) {
+ Stopwatch w= new Stopwatch().start();
+ System.out.println("STARTITER:" + i);
+ exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while (exec.next()) {
+ for (ValueVector v : exec) {
+ v.clear();
+ }
+ }
+ System.out.println("ENDITER: " + i);
+ System.out.println("TIME: " + w.elapsed(TimeUnit.MILLISECONDS) + "ms");
+ exec.stop();
+ }
+ context.close();
+ bit.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
index 9090685..e678c29 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -70,6 +71,7 @@ public class DumpCatTest extends ExecTest{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -87,6 +89,8 @@ public class DumpCatTest extends ExecTest{
}
assertTrue(!context.isFailed());
+ exec.stop();
+
FragmentHandle handle = context.getHandle();
/* Form the file name to which the trace output will dump the record batches */
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index e1fefdf..896cf4d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.compile;
import java.io.IOException;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.cache.local.LocalCache;
import org.apache.drill.exec.compile.ClassTransformer.ClassSet;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
@@ -72,7 +73,7 @@ public class TestClassTransformation extends BaseTestQuery {
public void testCompilationNoDebug() throws CompileException, ClassNotFoundException, ClassTransformationException, IOException {
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName());
- String sourceCode = cg.generate();
+ String sourceCode = cg.generateAndGet();
sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JDK.name()));
sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_DEBUG_OPTION, false));
@@ -104,9 +105,9 @@ public class TestClassTransformation extends BaseTestQuery {
private void compilationInnerClass(QueryClassLoader loader) throws Exception{
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
- ClassTransformer ct = new ClassTransformer();
- ExampleInner t = ct.getImplementationClass(loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName());
-
+ ClassTransformer ct = new ClassTransformer(new LocalCache());
+ Class<? extends ExampleInner> c = (Class<? extends ExampleInner>) ct.getImplementationClass(loader, cg.getDefinition(), cg.generateAndGet(), cg.getMaterializedClassName());
+ ExampleInner t = (ExampleInner) c.newInstance();
t.doOutside();
t.doInsideOutside();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
index 4d0cbcd..2265347 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
@@ -69,7 +69,7 @@ public class TestEvaluationVisitor {
ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
v.addExpr(e2, g.getRoot());
- System.out.println(g.generate());
+ System.out.println(g.generateAndGet());
}
private LogicalExpression getExpr(String expr) throws Exception{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
new file mode 100644
index 0000000..c7467a0
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.URL;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.compile.QueryClassLoader;
+import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.util.ASMifier;
+import org.objectweb.asm.util.CheckClassAdapter;
+import org.objectweb.asm.util.Textifier;
+import org.objectweb.asm.util.TraceClassVisitor;
+
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+
+public class ReplaceMethodInvoke {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReplaceMethodInvoke.class);
+
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws Exception{
+ String e = "org/apache/drill/ExampleReplaceable.class";
+ String h = "org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.class";
+ String k = "org/apache/drill/Pickle$OutgoingBatch.class";
+ String k2 = "org/apache/drill/Pickle.class";
+ String r = "org/apache/drill/exec/test/generated/FiltererGen0.class";
+ String p = "org/apache/drill/exec/test/generated/PartitionerGen4.class";
+ URL url = Resources.getResource(k2);
+ byte[] clazz = Resources.toByteArray(url);
+ ClassReader cr = new ClassReader(clazz);
+
+ ClassWriter cw = writer();
+ TraceClassVisitor visitor = new TraceClassVisitor(cw, new Textifier(), new PrintWriter(System.out));
+ //getTracer(false)
+ ValueHolderReplacementVisitor v2 = new ValueHolderReplacementVisitor(visitor);
+ cr.accept(v2, ClassReader.EXPAND_FRAMES );//| ClassReader.SKIP_DEBUG);
+
+ byte[] output = cw.toByteArray();
+ Files.write(output, new File("/src/scratch/bytes/S.class"));
+ check(output);
+
+
+ DrillConfig c = DrillConfig.createClient();
+ SystemOptionManager m = new SystemOptionManager(c, new LocalPStoreProvider(c));
+ m.init();
+ QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m);
+ ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output);
+ Class<?> clz = ql.loadClass("org.apache.drill.Pickle$OutgoingBatch");
+ clz.getMethod("x").invoke(null);
+
+ }
+
+
+ private static final void check(byte[] b) {
+ ClassReader cr = new ClassReader(b);
+ ClassWriter cw = writer();
+ ClassVisitor cv = new CheckClassAdapter(cw);
+ cr.accept(cv, 0);
+
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ CheckClassAdapter.verify(new ClassReader(cw.toByteArray()), false, pw);
+
+ assert sw.toString().length() == 0 : sw.toString();
+ }
+
+ private static ClassWriter writer() {
+ ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
+ return cw;
+ }
+
+ private static ClassVisitor getTracer(boolean asm) {
+ if (asm) {
+ return new TraceClassVisitor(null, new ASMifier(), new PrintWriter(System.out));
+ } else {
+ return new TraceClassVisitor(null, new Textifier(), new PrintWriter(System.out));
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index da82998..392af18 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -126,7 +126,7 @@ public class ExpressionTest extends ExecTest {
ClassGenerator<Projector> cg = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, new FunctionImplementationRegistry(DrillConfig.create())).getRoot();
cg.addExpr(new ValueVectorWriteExpression(new TypedFieldId(materializedExpr.getMajorType(), -1), materializedExpr));
- return cg.getCodeGenerator().generate();
+ return cg.getCodeGenerator().generateAndGet();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
new file mode 100644
index 0000000..632f5a4
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.fn.impl;
+
+import static org.junit.Assert.*;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.vector.ValueHolderHelper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestByteComparisonFunctions extends ExecTest{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestByteComparisonFunctions.class);
+
+ static TopLevelAllocator allocator;
+ static VarCharHolder hello;
+ static VarCharHolder goodbye;
+ static VarCharHolder helloLong;
+ static VarCharHolder goodbyeLong;
+
+ @BeforeClass
+ public static void setup(){
+ DrillConfig c= DrillConfig.create();
+ allocator = new TopLevelAllocator(c);
+ hello = ValueHolderHelper.getVarCharHolder(allocator, "hello");
+ goodbye = ValueHolderHelper.getVarCharHolder(allocator, "goodbye");
+ helloLong = ValueHolderHelper.getVarCharHolder(allocator, "hellomyfriend");
+ goodbyeLong = ValueHolderHelper.getVarCharHolder(allocator, "goodbyemyenemy");
+ }
+
+ @AfterClass
+ public static void teardown(){
+ hello.buffer.release();
+ helloLong.buffer.release();
+ goodbye.buffer.release();
+ goodbyeLong.buffer.release();
+ allocator.close();
+ }
+
+ @Test
+ public void testAfter(){
+ VarCharHolder left = hello;
+ VarCharHolder right = goodbye;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testBefore(){
+ VarCharHolder left = goodbye;
+ VarCharHolder right = hello;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == -1);
+ }
+
+ @Test
+ public void testEqualCompare(){
+ VarCharHolder left = hello;
+ VarCharHolder right = hello;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+
+ @Test
+ public void testEqual(){
+ VarCharHolder left = hello;
+ VarCharHolder right = hello;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testNotEqual(){
+ VarCharHolder left = hello;
+ VarCharHolder right = goodbye;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+
+ @Test
+ public void testAfterLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = goodbyeLong;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testBeforeLong(){
+ VarCharHolder left = goodbyeLong;
+ VarCharHolder right = helloLong;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == -1);
+ }
+
+ @Test
+ public void testEqualCompareLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = helloLong;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+
+ @Test
+ public void testEqualLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = helloLong;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testNotEqualLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = goodbyeLong;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
index 104923b..4f06a9d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -64,6 +65,7 @@ public class TestMathFunctions extends ExecTest {
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
index 62a224e..3d41f0b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
@@ -27,6 +27,7 @@ import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -83,6 +84,7 @@ public class TestNewMathFunctions {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
index 4345560..fd71dd9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
@@ -27,6 +27,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -64,6 +65,7 @@ public class TestRepeatedFunction extends ExecTest{
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
[06/47] git commit: Skip static fields in PojoDataType.
Posted by ja...@apache.org.
Skip static fields in PojoDataType.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ed72c137
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ed72c137
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ed72c137
Branch: refs/heads/master
Commit: ed72c1370fe98de28bebc587e063e1728a8027c3
Parents: fab96c3
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Aug 24 08:28:25 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Aug 24 08:28:25 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/store/pojo/PojoDataType.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed72c137/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoDataType.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoDataType.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoDataType.java
index d92ac1c..c84f43c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoDataType.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoDataType.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.pojo;
import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
import java.sql.Timestamp;
import java.util.List;
@@ -39,6 +40,8 @@ public class PojoDataType {
for(int i = 0; i < fields.length; i++){
Field f = fields[i];
+ if(Modifier.isStatic(f.getModifiers())) continue;
+
Class<?> type = f.getType();
names.add(f.getName());
[44/47] git commit: DRILL-1283: JSON project pushdown.
Posted by ja...@apache.org.
DRILL-1283: JSON project pushdown.
Allows for users to avoid reading columns of a JSON file, including those that include elements of JSON that drill does not currently support. This can be used to query a subset of an existing file while avoiding elements like schema changes in some columns or nulls in lists that are currently not compatible with Drill.
Patch was revised based on Hanifi's review comments, and then rebased off of the merge branch.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/929d765a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/929d765a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/929d765a
Branch: refs/heads/master
Commit: 929d765afd9da2fb0010a97e90b2ee19f245e37c
Parents: cc25504
Author: Jason Altekruse <al...@gmail.com>
Authored: Tue Aug 5 15:44:42 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 18:38:32 2014 -0700
----------------------------------------------------------------------
.../drill/common/expression/PathSegment.java | 42 +
.../drill/common/expression/SchemaPath.java | 16 +
.../src/main/codegen/templates/BaseWriter.java | 4 +
.../main/codegen/templates/ComplexWriters.java | 13 +-
.../codegen/templates/FixedValueVectors.java | 2 +-
.../src/main/codegen/templates/ListWriters.java | 10 +-
.../src/main/codegen/templates/MapWriters.java | 12 +-
.../exec/expr/fn/impl/conv/JsonConvertFrom.java | 4 +-
.../exec/store/easy/json/JSONFormatPlugin.java | 2 +-
.../exec/store/easy/json/JSONRecordReader2.java | 42 +-
.../drill/exec/vector/BaseValueVector.java | 1 +
.../org/apache/drill/exec/vector/BitVector.java | 2 +-
.../exec/vector/complex/fn/JsonReader.java | 130 +-
.../vector/complex/fn/JsonReaderWithState.java | 14 +-
.../vector/complex/impl/ComplexWriterImpl.java | 9 +-
.../complex/impl/VectorContainerWriter.java | 8 +
.../java/org/apache/drill/BaseTestQuery.java | 2 +-
.../physical/impl/writer/TestParquetWriter.java | 3 +
.../vector/complex/writer/TestJsonReader.java | 160 +-
.../project_pushdown_json_physical_plan.json | 50 +
.../store/json/schema_change_int_to_string.json | 30 +
.../store/json/single_column_long_file.json | 13512 +++++++++++++++++
.../store/json/test_complex_read_with_star.json | 24 +
23 files changed, 14049 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
index 0ecfcd0..c434dc7 100644
--- a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -239,4 +239,46 @@ public abstract class PathSegment{
} else return child.equals(other.child);
}
+ /**
+ * Check if another path is contained in this one. This is useful for 2 cases. The first
+ * is checking if the other is lower down in the tree, below this path. The other is if
+ * a path is actually contained above the current one.
+ *
+ * Examples:
+ * [a] . contains( [a.b.c] ) returns true
+ * [a.b.c] . contains( [a] ) returns true
+ *
+ * This behavior is used for cases like scanning json in an event based fashion, when we arrive at
+ * a node in a complex type, we will know the complete path back to the root. This method can
+ * be used to determine if we need the data below. This is true in both the cases where the
+ * column requested from the user is below the current node (in which case we may ignore other nodes
+ * further down the tree, while keeping others). This is also the case if the requested path is further
+ * up the tree, if we know we are at position a.b.c and a.b was a requested column, we need to scan
+ * all of the data at and below the current a.b.c node.
+ *
+ * @param otherSeg - path segment to check if it is contained below this one.
+ * @return - is this a match
+ */
+ public boolean contains(PathSegment otherSeg) {
+ if (this == otherSeg)
+ return true;
+ if (otherSeg == null)
+ return false;
+ // TODO - fix this in the future to match array segments are part of the path
+ // the current behavior to always return true when we hit an array may be useful in some cases,
+ // but we can get better performance in the JSON reader if we avoid reading unwanted elements in arrays
+ if (otherSeg.isArray() || this.isArray())
+ return true;
+ if (getClass() != otherSeg.getClass())
+ return false;
+
+ if (!segmentEquals(otherSeg)) {
+ return false;
+ }
+ else if (child == null || otherSeg.child == null) {
+ return true;
+ } else return child.contains(otherSeg.child);
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 25ee8b4..9f444e4 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -199,6 +199,22 @@ public class SchemaPath extends LogicalExpressionBase {
return rootSegment.equals(other.rootSegment);
}
+ public boolean contains(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (!(obj instanceof SchemaPath))
+ return false;
+
+ SchemaPath other = (SchemaPath) obj;
+ if (rootSegment == null) {
+ return true;
+ }
+ return rootSegment.contains(other.rootSegment);
+
+ }
+
@Override
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/codegen/templates/BaseWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/BaseWriter.java b/exec/java-exec/src/main/codegen/templates/BaseWriter.java
index e281bc7..69ab6cd 100644
--- a/exec/java-exec/src/main/codegen/templates/BaseWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/BaseWriter.java
@@ -33,6 +33,10 @@ public interface BaseWriter extends Positionable{
WriteState getState();
public interface MapWriter extends BaseWriter{
+
+ MaterializedField getField();
+ void checkValueCapacity();
+
<#list vv.types as type><#list type.minor as minor>
<#assign lowerName = minor.class?uncap_first />
<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java b/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
index c390770..2442434 100644
--- a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
+++ b/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
@@ -40,6 +40,7 @@ package org.apache.drill.exec.vector.complex.impl;
<#include "/@includes/vv_imports.ftl" />
+/* This class is generated using freemarker and the ComplexWriters.java template */
@SuppressWarnings("unused")
public class ${eName}WriterImpl extends AbstractFieldWriter {
@@ -51,7 +52,15 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
this.mutator = vector.getMutator();
this.vector = vector;
}
-
+
+ public MaterializedField getField(){
+ return vector.getField();
+ }
+
+ public void checkValueCapacity() {
+ inform(vector.getValueCapacity() > idx());
+ }
+
public void allocate(){
inform(vector.allocateNewSafe());
}
@@ -111,7 +120,7 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
vector.setCurrentValueCount(idx());
}
}
-
+
</#if>
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index 8b4496e..58e6ccc 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -44,7 +44,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
- private int allocationValueCount = 4096;
+ private int allocationValueCount = INITIAL_VALUE_ALLOCATION;
private int allocationMonitor = 0;
public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/codegen/templates/ListWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ListWriters.java b/exec/java-exec/src/main/codegen/templates/ListWriters.java
index 278fddc..c991fcc 100644
--- a/exec/java-exec/src/main/codegen/templates/ListWriters.java
+++ b/exec/java-exec/src/main/codegen/templates/ListWriters.java
@@ -36,7 +36,7 @@ package org.apache.drill.exec.vector.complex.impl;
<#include "/@includes/vv_imports.ftl" />
-
+/* This class is generated using freemarker and the ListWriters.java template */
@SuppressWarnings("unused")
public class ${mode}ListWriter extends AbstractFieldWriter{
@@ -136,6 +136,14 @@ public class ${mode}ListWriter extends AbstractFieldWriter{
}
</#list></#list>
+ public MaterializedField getField() {
+ return container.getField();
+ }
+
+ public void checkValueCapacity() {
+ inform(container.getValueCapacity() > idx());
+ }
+
<#if mode == "Repeated">
public void start(){
if(ok()){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java
index 7aa5a2e..dd7e653 100644
--- a/exec/java-exec/src/main/codegen/templates/MapWriters.java
+++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.vector.complex.writer.FieldWriter;
import com.google.common.collect.Maps;
+/* This class is generated using freemarker and the MapWriters.java template */
@SuppressWarnings("unused")
public class ${mode}MapWriter extends AbstractFieldWriter{
@@ -53,7 +54,14 @@ public class ${mode}MapWriter extends AbstractFieldWriter{
this.container = container;
}
-
+ public MaterializedField getField() {
+ return container.getField();
+ }
+
+ public void checkValueCapacity(){
+ inform(container.getValueCapacity() > idx());
+ }
+
public MapWriter map(String name){
FieldWriter writer = fields.get(name);
if(writer == null){
@@ -153,7 +161,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter{
FieldWriter writer = fields.get(name);
if(writer == null){
${vectName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
- AllocationHelper.allocate(vector, 1000, 100, 10);
+ vector.allocateNewSafe();
writer = new ${vectName}WriterImpl(vector, this);
writer.setPosition(${index});
fields.put(name, writer);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
index 53b8f50..e66cabb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
@@ -67,7 +67,7 @@ public class JsonConvertFrom {
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
try {
- org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer);
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer, null);
jsonReader.write(new java.io.StringReader(input), writer);
@@ -94,7 +94,7 @@ public class JsonConvertFrom {
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
try {
- org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer);
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer, null);
jsonReader.write(new java.io.StringReader(input), writer);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
index 8b5577c..4adac20 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -117,7 +117,7 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
@Override
public boolean supportsPushDown() {
- return false;
+ return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
index 5d741e0..5bfc482 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
@@ -24,6 +24,7 @@ import java.util.List;
import com.fasterxml.jackson.core.JsonParseException;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.OutOfMemoryException;
@@ -31,11 +32,14 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.complex.fn.JsonReader;
import org.apache.drill.exec.vector.complex.fn.JsonReaderWithState;
import org.apache.drill.exec.vector.complex.fn.JsonRecordSplitter;
import org.apache.drill.exec.vector.complex.fn.UTF8JsonRecordSplitter;
import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -53,13 +57,14 @@ public class JSONRecordReader2 implements RecordReader{
private int recordCount;
private FragmentContext fragmentContext;
private OperatorContext operatorContext;
-
+ private List<SchemaPath> columns;
public JSONRecordReader2(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem,
List<SchemaPath> columns) throws OutOfMemoryException {
this.hadoopPath = new Path(inputPath);
this.fileSystem = fileSystem;
this.fragmentContext = fragmentContext;
+ this.columns = columns;
}
@Override
@@ -69,9 +74,9 @@ public class JSONRecordReader2 implements RecordReader{
JsonRecordSplitter splitter = new UTF8JsonRecordSplitter(stream);
this.writer = new VectorContainerWriter(output);
this.mutator = output;
- jsonReader = new JsonReaderWithState(splitter, fragmentContext.getManagedBuffer());
+ jsonReader = new JsonReaderWithState(splitter, fragmentContext.getManagedBuffer(), columns);
}catch(Exception e){
- throw new ExecutionSetupException("Failure reading JSON file.", e);
+ handleAndRaise("Failure reading JSON file.", e);
}
}
@@ -102,7 +107,7 @@ public class JSONRecordReader2 implements RecordReader{
recordCount = 0;
try{
- outside: while(true){
+ outside: while(true && recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){
writer.setPosition(recordCount);
switch(jsonReader.write(writer)){
@@ -120,12 +125,31 @@ public class JSONRecordReader2 implements RecordReader{
break outside;
};
}
- } catch(Exception e) {
- handleAndRaise("Failure while parsing JSON file.", e);
- }
+ for (SchemaPath sp :jsonReader.getNullColumns() ) {
+ PathSegment root = sp.getRootSegment();
+ BaseWriter.MapWriter fieldWriter = writer.rootAsMap();
+ if (root.getChild() != null && ! root.getChild().isArray()) {
+ fieldWriter = fieldWriter.map(root.getNameSegment().getPath());
+ while ( root.getChild().getChild() != null && ! root.getChild().isArray() ) {
+ fieldWriter = fieldWriter.map(root.getChild().getNameSegment().getPath());
+ root = root.getChild();
+ }
+ fieldWriter.integer(root.getChild().getNameSegment().getPath());
+ } else {
+ fieldWriter.integer(root.getNameSegment().getPath());
+ }
+ }
- writer.setValueCount(recordCount);
- return recordCount;
+ writer.setValueCount(recordCount);
+ return recordCount;
+
+ } catch (JsonParseException e) {
+ handleAndRaise("Error parsing JSON.", e);
+ } catch (IOException e) {
+ handleAndRaise("Error reading JSON.", e);
+ }
+ // this is never reached
+ return 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index cf62d54..1b3705e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -34,6 +34,7 @@ public abstract class BaseValueVector implements ValueVector{
protected final BufferAllocator allocator;
protected final MaterializedField field;
+ public static final int INITIAL_VALUE_ALLOCATION = 4096;
BaseValueVector(MaterializedField field, BufferAllocator allocator) {
this.allocator = allocator;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 0076a1d..82cc4c9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -41,7 +41,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
- private int allocationValueCount = 4096;
+ private int allocationValueCount = INITIAL_VALUE_ALLOCATION;
private int allocationMonitor = 0;
private int valueCapacity;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 59035cb..8393dc6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -21,11 +21,17 @@ import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.Float8Holder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
@@ -46,11 +52,65 @@ public class JsonReader {
private final JsonFactory factory = new JsonFactory();
private JsonParser parser;
private DrillBuf workBuf;
+ private List<SchemaPath> columns;
+ // This is a parallel array for the field above to indicate if we have found any values in a
+ // given selected column. This allows for columns that are requested to receive a vector full of
+ // null values if no values were found in an entire read. The reason this needs to happen after
+ // all of the records have been read in a batch is to prevent a schema change when we actually find
+ // data in that column.
+ private boolean[] columnsFound;
+ // A flag set at setup time if the start column is in the requested column list, prevents
+ // doing a more computational intensive check if we are supposed to be reading a column
+ private boolean starRequested;
- public JsonReader(DrillBuf managedBuf) throws JsonParseException, IOException {
+ public JsonReader() throws IOException {
+ this(null, null);
+ }
+
+ public JsonReader(DrillBuf managedBuf, List<SchemaPath> columns) throws JsonParseException, IOException {
factory.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
factory.configure(Feature.ALLOW_COMMENTS, true);
this.workBuf = managedBuf;
+ this.columns = columns;
+ // TODO - remove this check once the optimizer is updated to push down * instead of a null list
+ if (this.columns == null) {
+ this.columns = new ArrayList();
+ this.columns.add(new SchemaPath(new PathSegment.NameSegment("*")));
+ }
+ this.columnsFound = new boolean[this.columns.size()];
+ this.starRequested = containsStar();
+ }
+
+ private boolean containsStar() {
+ for (SchemaPath expr : this.columns){
+ if (expr.getRootSegment().getPath().equals("*"))
+ return true;
+ }
+ return false;
+ }
+
+ private boolean fieldSelected(SchemaPath field){
+ if (starRequested)
+ return true;
+ int i = 0;
+ for (SchemaPath expr : this.columns){
+ if ( expr.contains(field)){
+ columnsFound[i] = true;
+ return true;
+ }
+ i++;
+ }
+ return false;
+ }
+
+ public List<SchemaPath> getNullColumns() {
+ ArrayList<SchemaPath> nullColumns = new ArrayList<SchemaPath>();
+ for (int i = 0; i < columnsFound.length; i++ ) {
+ if ( ! columnsFound[i] && ! columns.get(i).getRootSegment().getPath().equals("*") ) {
+ nullColumns.add(columns.get(i));
+ }
+ }
+ return nullColumns;
}
public boolean write(Reader reader, ComplexWriter writer) throws JsonParseException, IOException {
@@ -81,6 +141,33 @@ public class JsonReader {
return true;
}
+ private void consumeEntireNextValue(JsonParser parser) throws IOException {
+ switch(parser.nextToken()){
+ case START_ARRAY:
+ case START_OBJECT:
+ int arrayAndObjectCounter = 1;
+ skipArrayLoop: while (true) {
+ switch(parser.nextToken()) {
+ case START_ARRAY:
+ case START_OBJECT:
+ arrayAndObjectCounter++;
+ break;
+ case END_ARRAY:
+ case END_OBJECT:
+ arrayAndObjectCounter--;
+ if (arrayAndObjectCounter == 0) {
+ break skipArrayLoop;
+ }
+ break;
+ }
+ }
+ break;
+ default:
+ // hit a single value, do nothing as the token was already read
+ // in the switch statement
+ break;
+ }
+ }
private void writeData(MapWriter map) throws JsonParseException, IOException {
//
@@ -91,7 +178,16 @@ public class JsonReader {
assert t == JsonToken.FIELD_NAME : String.format("Expected FIELD_NAME but got %s.", t.name());
final String fieldName = parser.getText();
-
+ SchemaPath path;
+ if (map.getField().getPath().getRootSegment().getPath().equals("")) {
+ path = new SchemaPath(new PathSegment.NameSegment(fieldName));
+ } else {
+ path = map.getField().getPath().getChild(fieldName);
+ }
+ if ( ! fieldSelected(path) ) {
+ consumeEntireNextValue(parser);
+ continue outside;
+ }
switch(parser.nextToken()){
case START_ARRAY:
@@ -117,6 +213,7 @@ public class JsonReader {
break;
}
case VALUE_NULL:
+ map.checkValueCapacity();
// do nothing as we don't have a type.
break;
case VALUE_NUMBER_FLOAT:
@@ -131,14 +228,7 @@ public class JsonReader {
break;
case VALUE_STRING:
VarCharHolder vh = new VarCharHolder();
- String value = parser.getText();
- byte[] b = value.getBytes(Charsets.UTF_8);
- ensure(b.length);
- workBuf.setBytes(0, b);
- vh.buffer = workBuf;
- vh.start = 0;
- vh.end = b.length;
- map.varChar(fieldName).write(vh);
+ map.varChar(fieldName).write(prepareVarCharHolder(vh, parser));
break;
default:
@@ -155,6 +245,17 @@ public class JsonReader {
workBuf = workBuf.reallocIfNeeded(length);
}
+ private VarCharHolder prepareVarCharHolder(VarCharHolder vh, JsonParser parser) throws IOException {
+ String value = parser.getText();
+ byte[] b = value.getBytes(Charsets.UTF_8);
+ ensure(b.length);
+ workBuf.setBytes(0, b);
+ vh.buffer = workBuf;
+ vh.start = 0;
+ vh.end = b.length;
+ return vh;
+ }
+
private void writeData(ListWriter list) throws JsonParseException, IOException {
list.start();
outside: while(true){
@@ -198,14 +299,7 @@ public class JsonReader {
break;
case VALUE_STRING:
VarCharHolder vh = new VarCharHolder();
- String value = parser.getText();
- byte[] b = value.getBytes(Charsets.UTF_8);
- ensure(b.length);
- workBuf.setBytes(0, b);
- vh.buffer = workBuf;
- vh.start = 0;
- vh.end = b.length;
- list.varChar().write(vh);
+ list.varChar().write(prepareVarCharHolder(vh, parser));
break;
default:
throw new IllegalStateException("Unexpected token " + parser.getCurrentToken());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
index 54e063b..13b8215 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
@@ -22,7 +22,9 @@ import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.io.Reader;
+import java.util.List;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import com.fasterxml.jackson.core.JsonParseException;
@@ -37,10 +39,18 @@ public class JsonReaderWithState {
private JsonRecordSplitter splitter;
private JsonReader jsonReader;
- public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace) throws IOException{
+ public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace, List<SchemaPath> columns) throws IOException{
this.splitter = splitter;
reader = splitter.getNextReader();
- jsonReader = new JsonReader(workspace);
+ jsonReader = new JsonReader(workspace, columns);
+ }
+
+ public JsonReaderWithState(JsonRecordSplitter splitter) throws IOException{
+ this(splitter, null, null);
+ }
+
+ public List<SchemaPath> getNullColumns() {
+ return jsonReader.getNullColumns();
}
public WriteState write(ComplexWriter writer) throws JsonParseException, IOException {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
index 2fa72f7..165c5b5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
@@ -48,6 +48,14 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
this.container = container;
}
+ public MaterializedField getField() {
+ return container.getField();
+ }
+
+ public void checkValueCapacity(){
+ inform(container.getValueCapacity() > idx());
+ }
+
private void check(Mode... modes){
StateTool.check(mode, modes);
}
@@ -164,7 +172,6 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
return listRoot;
}
-
private static class VectorAccessibleFacade extends MapVector {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
index 4f669c0..57980f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
@@ -40,6 +40,14 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple
this.mapRoot = new SingleMapWriter(mapVector, this);
}
+ public MaterializedField getField() {
+ return mapVector.getField();
+ }
+
+ public void checkValueCapacity(){
+ inform(mapVector.getValueCapacity() > idx());
+ }
+
public MapVector getMapVector() {
return mapVector;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 5e52e82..f504de4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -147,7 +147,7 @@ public class BaseTestQuery extends ExecTest{
return client.runQuery(type, query);
}
- private int testRunAndPrint(QueryType type, String query) throws Exception{
+ protected int testRunAndPrint(QueryType type, String query) throws Exception{
query = query.replace("[WORKING_PATH]", TestTools.getWorkingPath());
PrintingResultsListener resultListener = new PrintingResultsListener(client.getConfig(), Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH);
client.runQuery(type, query, resultListener);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index b4fd09d..0f0743a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
@@ -198,6 +199,8 @@ public class TestParquetWriter extends BaseTestQuery {
//test(String.format("ALTER SESSION SET `%s` = %d", ExecConstants.PARQUET_BLOCK_SIZE, 1*1024*1024));
String selection = "mi";
String inputTable = "cp.`customer.json`";
+ int count = testRunAndPrint(UserBitShared.QueryType.SQL, "select mi from cp.`customer.json`");
+ System.out.println(count);
runTestAndValidate(selection, selection, inputTable, "foodmart_customer_parquet");
} finally {
test(String.format("ALTER SESSION SET `%s` = %d", ExecConstants.PARQUET_BLOCK_SIZE, 512*1024*1024));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index f2f91f4..885e50a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -17,14 +17,32 @@
*/
package org.apache.drill.exec.vector.complex.writer;
+import static org.jgroups.util.Util.assertTrue;
import static org.junit.Assert.assertEquals;
import io.netty.buffer.DrillBuf;
+import static org.junit.Assert.assertNull;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.util.List;
+import com.google.common.io.Files;
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.RepeatedBigIntVector;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.fn.JsonReader;
import org.apache.drill.exec.vector.complex.fn.JsonReaderWithState;
@@ -34,6 +52,7 @@ import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -41,10 +60,11 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
-public class TestJsonReader {
+public class TestJsonReader extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJsonReader.class);
private static BufferAllocator allocator;
+ private static final boolean VERBOSE_DEBUG = false;
@BeforeClass
public static void setupAllocator(){
@@ -56,6 +76,142 @@ public class TestJsonReader {
allocator.close();
}
+ public void runTestsOnFile(String filename, UserBitShared.QueryType queryType, String[] queries, long[] rowCounts) throws Exception {
+ if (VERBOSE_DEBUG) {
+ System.out.println("===================");
+ System.out.println("source data in json");
+ System.out.println("===================");
+ System.out.println(Files.toString(FileUtils.getResourceAsFile(filename), Charsets.UTF_8));
+ }
+
+ int i = 0;
+ for (String query : queries) {
+ if (VERBOSE_DEBUG) {
+ System.out.println("=====");
+ System.out.println("query");
+ System.out.println("=====");
+ System.out.println(query);
+ System.out.println("======");
+ System.out.println("result");
+ System.out.println("======");
+ }
+ int rowCount = testRunAndPrint(queryType, query);
+ assertEquals( rowCount, rowCounts[i]);
+ System.out.println();
+ i++;
+ }
+ }
+
+ @Test
+ public void testSingleColumnRead_vector_fill_bug() throws Exception {
+ String[] queries = {"select * from cp.`/store/json/single_column_long_file.json`"};
+ long[] rowCounts = {13512};
+ String filename = "/store/json/single_column_long_file.json";
+ runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
+ }
+
+ @Test
+ public void testNonExistentColumnReadAlone() throws Exception {
+ String[] queries = {"select non_existent_column from cp.`/store/json/single_column_long_file.json`"};
+ long[] rowCounts = {13512};
+ String filename = "/store/json/single_column_long_file.json";
+ runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
+ }
+
+ @Test
+ public void readComplexWithStar() throws Exception {
+ List<QueryResultBatch> results = testSqlWithResults("select * from cp.`/store/json/test_complex_read_with_star.json`");
+ assertEquals(2, results.size());
+
+ RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
+ QueryResultBatch batch = results.get(0);
+
+ assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
+ assertEquals(3, batchLoader.getSchema().getFieldCount());
+ testExistentColumns(batchLoader, batch);
+
+ batch.release();
+ batchLoader.clear();
+ }
+
+ // The project pushdown rule is correctly adding the projected columns to the scan, however it is not removing
+ // the redundant project operator after the scan, this tests runs a physical plan generated from one of the tests to
+ // ensure that the project is filtering out the correct data in the scan alone
+ @Test
+ public void testProjectPushdown() throws Exception {
+ String[] queries = {Files.toString(FileUtils.getResourceAsFile("/store/json/project_pushdown_json_physical_plan.json"), Charsets.UTF_8)};
+ long[] rowCounts = {3};
+ String filename = "/store/json/schema_change_int_to_string.json";
+ runTestsOnFile(filename, UserBitShared.QueryType.PHYSICAL, queries, rowCounts);
+
+ List<QueryResultBatch> results = testPhysicalWithResults(queries[0]);
+ assertEquals(2, results.size());
+ // "`field_1`", "`field_3`.`inner_1`", "`field_3`.`inner_2`", "`field_4`.`inner_1`"
+
+ RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
+ QueryResultBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
+ assertEquals(5, batchLoader.getSchema().getFieldCount());
+ testExistentColumns(batchLoader, batch);
+
+ VectorWrapper vw = batchLoader.getValueAccessorById(
+ NullableIntVector.class, //
+ batchLoader.getValueVectorId(SchemaPath.getCompoundPath("non_existent_at_root")).getFieldIds() //
+ );
+ assertNull(vw.getValueVector().getAccessor().getObject(0));
+ assertNull(vw.getValueVector().getAccessor().getObject(1));
+ assertNull(vw.getValueVector().getAccessor().getObject(2));
+
+ vw = batchLoader.getValueAccessorById(
+ NullableIntVector.class, //
+ batchLoader.getValueVectorId(SchemaPath.getCompoundPath("non_existent", "nested","field")).getFieldIds() //
+ );
+ assertNull(vw.getValueVector().getAccessor().getObject(0));
+ assertNull(vw.getValueVector().getAccessor().getObject(1));
+ assertNull(vw.getValueVector().getAccessor().getObject(2));
+
+ vw.getValueVector().clear();
+ batch.release();
+ batchLoader.clear();
+ }
+
+ private void testExistentColumns(RecordBatchLoader batchLoader, QueryResultBatch batch) throws SchemaChangeException {
+
+ assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
+
+ VectorWrapper<?> vw = batchLoader.getValueAccessorById(
+ RepeatedBigIntVector.class, //
+ batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_1")).getFieldIds() //
+ );
+ assertEquals("[1]", vw.getValueVector().getAccessor().getObject(0).toString());
+ assertEquals("[5]", vw.getValueVector().getAccessor().getObject(1).toString());
+ assertEquals("[5,10,15]", vw.getValueVector().getAccessor().getObject(2).toString());
+
+ vw = batchLoader.getValueAccessorById(
+ IntVector.class, //
+ batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_3", "inner_1")).getFieldIds() //
+ );
+ assertNull(vw.getValueVector().getAccessor().getObject(0));
+ assertEquals(2l, vw.getValueVector().getAccessor().getObject(1));
+ assertEquals(5l, vw.getValueVector().getAccessor().getObject(2));
+
+ vw = batchLoader.getValueAccessorById(
+ IntVector.class, //
+ batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_3", "inner_2")).getFieldIds() //
+ );
+ assertNull(vw.getValueVector().getAccessor().getObject(0));
+ assertNull(vw.getValueVector().getAccessor().getObject(1));
+ assertEquals(3l, vw.getValueVector().getAccessor().getObject(2));
+
+ vw = batchLoader.getValueAccessorById(
+ RepeatedBigIntVector.class, //
+ batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_4", "inner_1")).getFieldIds() //
+ );
+ assertEquals("[]", vw.getValueVector().getAccessor().getObject(0).toString());
+ assertEquals("[1,2,3]", vw.getValueVector().getAccessor().getObject(1).toString());
+ assertEquals("[4,5,6]", vw.getValueVector().getAccessor().getObject(2).toString());
+ }
+
@Test
public void testReader() throws Exception{
final int repeatSize = 10;
@@ -89,7 +245,7 @@ public class TestJsonReader {
writer.allocate();
DrillBuf buffer = allocator.buffer(255);
- JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound), buffer);
+ JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound), buffer, null);
int i =0;
List<Integer> batchSizes = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/resources/store/json/project_pushdown_json_physical_plan.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/project_pushdown_json_physical_plan.json b/exec/java-exec/src/test/resources/store/json/project_pushdown_json_physical_plan.json
new file mode 100644
index 0000000..60cd2b0
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/project_pushdown_json_physical_plan.json
@@ -0,0 +1,50 @@
+{
+ "head" : {
+ "version" : 1,
+ "generator" : {
+ "type" : "DefaultSqlHandler",
+ "info" : ""
+ },
+ "type" : "APACHE_DRILL_PHYSICAL",
+ "options" : [ {
+ "name" : "planner.width.max_per_node",
+ "kind" : "LONG",
+ "type" : "SESSION",
+ "num_val" : 2
+ } ],
+ "queue" : 0,
+ "resultMode" : "EXEC"
+ },
+ "graph" : [ {
+ "pop" : "fs-scan",
+ "@id" : 1,
+ "files" : [ "/store/json/schema_change_int_to_string.json" ],
+ "storage" : {
+ "type" : "file",
+ "enabled" : true,
+ "connection" : "classpath:///",
+ "workspaces" : null,
+ "formats" : {
+ "json" : {
+ "type" : "json"
+ },
+ "parquet" : {
+ "type" : "parquet"
+ }
+ }
+ },
+ "format" : {
+ "type" : "json"
+ },
+ "columns" : [ "`field_1`", "`field_3`.`inner_1`", "`field_3`.`inner_2`", "`field_4`.`inner_1`", "`non_existent_at_root`", "`non_existent`.`nested`.`field`"],
+ "selectionRoot" : "/store/json/schema_change_int_to_string.json",
+ "cost" : 0.0
+ }, {
+ "pop" : "screen",
+ "@id" : 0,
+ "child" : 1,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 1.0
+ } ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json b/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
new file mode 100644
index 0000000..f2fca86
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
@@ -0,0 +1,30 @@
+{
+ "field_1": [1]
+}
+{
+ "field_1": [5],
+ "field_2": 2,
+ "field_3": {
+ "inner_1" : 2
+ },
+ "field_4" : {
+ "inner_1" : [1,2,3],
+ "inner_2" : 3,
+ "inner_3" : { "inner_object_field_1" : 2}
+ },
+ "field_5" : [ { "inner_list" : [1,6] }, { "inner_list":[3,8]}, { "inner_list":[12,4,5]} ]
+}
+{
+ "field_1": [5,10,15],
+ "field_2": "A wild string appears!",
+ "field_3": {
+ "inner_1" : 5,
+ "inner_2" : 3,
+ "inner_3" : [ { "inner_object_field_1" : 2}, {"inner_object_field_1" : 10} ]
+ },
+ "field_4" : {
+ "inner_1" : [4,5,6],
+ "inner_2" : 3
+ },
+ "field_5" : [ { "inner_list" : [5,6.0, "1234"] }, { "inner_list":[7,8.0, "12341324"], "inner_list_2" : [1,2,2323.443e10, "hello there"]}, { "inner_list":[3,4,5], "inner_list_2" : [10, 11, 12]} ]
+}
[42/47] DRILL-1283: JSON project pushdown.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/resources/store/json/test_complex_read_with_star.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/test_complex_read_with_star.json b/exec/java-exec/src/test/resources/store/json/test_complex_read_with_star.json
new file mode 100644
index 0000000..4256600
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/test_complex_read_with_star.json
@@ -0,0 +1,24 @@
+{
+ "field_1": [1]
+}
+{
+ "field_1": [5],
+ "field_3": {
+ "inner_1" : 2
+ },
+ "field_4" : {
+ "inner_1" : [1,2,3],
+ "inner_2" : 3
+ }
+}
+{
+ "field_1": [5,10,15],
+ "field_3": {
+ "inner_1" : 5,
+ "inner_2" : 3
+ },
+ "field_4" : {
+ "inner_1" : [4,5,6],
+ "inner_2" : 3
+ }
+}
[34/47] git commit: DRILL-1337 : Fix incorrect query result for TPCH
Q13. Stop pulling up local filter from an outer join condition.
Posted by ja...@apache.org.
DRILL-1337 : Fix incorrect query result for TPCH Q13. Stop pulling up local filter from an outer join condition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/746a0c7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/746a0c7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/746a0c7a
Branch: refs/heads/master
Commit: 746a0c7a6a151cfe97d7f69a4b3395eaf6b5e4b8
Parents: 4f59d4f
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Mon Aug 25 17:08:49 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 13:33:47 2014 -0700
----------------------------------------------------------------------
.../drill/exec/planner/logical/DrillJoinRule.java | 5 ++++-
.../test/java/org/apache/drill/TestBugFixes.java | 15 +++++++++++++++
.../java/org/apache/drill/TestExampleQueries.java | 16 ++++++++--------
3 files changed, 27 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/746a0c7a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
index b861897..cf15411 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
@@ -22,6 +22,7 @@ import java.util.logging.Logger;
import org.eigenbase.rel.InvalidRelException;
import org.eigenbase.rel.JoinRel;
+import org.eigenbase.rel.JoinRelType;
import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.Convention;
import org.eigenbase.relopt.RelOptRule;
@@ -73,8 +74,10 @@ public class DrillJoinRule extends RelOptRule {
// If the join involves equijoins and non-equijoins, then we can process the non-equijoins through
// a filter right after the join
+ // DRILL-1337: We can only pull up a non-equivjoin filter for INNER join.
+ // For OUTER join, pulling up a non-eqivjoin filter will lead to incorrectly discarding qualified rows.
if (! remaining.isAlwaysTrue()) {
- if (hasEquijoins) {
+ if (hasEquijoins && join.getJoinType()== JoinRelType.INNER) {
addFilter = true;
List<RexNode> equijoinList = Lists.newArrayList();
List<RelDataTypeField> leftTypes = convertedLeft.getRowType().getFieldList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/746a0c7a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
index e6d698a..743aeae 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill;
+import org.apache.drill.exec.rpc.RpcException;
import org.junit.Ignore;
import org.junit.Test;
@@ -60,4 +61,18 @@ public class TestBugFixes extends BaseTestQuery {
String query = "select sum(cast(employee_id as decimal(38, 18))), avg(cast(employee_id as decimal(38, 18))) from cp.`employee.json` group by (department_id)";
test(query);
}
+
+
+ @Test (expected = RpcException.class)
+ // Should be "Failure while parsing sql. Node [rel#26:Subset#6.LOGICAL.ANY([]).[]] could not be implemented;".
+ // Drill will hit CanNotPlan, until we add code fix to transform the local LHS filter in left outer join properly.
+ public void testDRILL1337_LocalLeftFilterLeftOutJoin() throws Exception {
+ test("select count(*) from cp.`tpch/nation.parquet` n left outer join cp.`tpch/region.parquet` r on n.n_regionkey = r.r_regionkey and n.n_nationkey > 10;");
+ }
+
+ @Test
+ public void testDRILL1337_LocalRightFilterLeftOutJoin() throws Exception {
+ test("select * from cp.`tpch/nation.parquet` n left outer join cp.`tpch/region.parquet` r on n.n_regionkey = r.r_regionkey and r.r_name not like '%ASIA' order by r.r_name;");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/746a0c7a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 0053388..57d4043 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -148,20 +148,20 @@ public class TestExampleQueries extends BaseTestQuery{
@Test
public void testPushExpInJoinConditionLeftJoin() throws Exception {
- test("select a.n_nationkey from cp.`tpch/nation.parquet` a left join cp.`tpch/region.parquet` b " + "" +
+ test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a left join cp.`tpch/region.parquet` b " + "" +
" on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter
- " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') " + // left filter
- " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') " + // right filter
- " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
+ // " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') " + // left filter
+ " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') ") ; // right filter
+ // " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
}
@Test
public void testPushExpInJoinConditionRightJoin() throws Exception {
- test("select a.n_nationkey from cp.`tpch/nation.parquet` a right join cp.`tpch/region.parquet` b " + "" +
+ test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a right join cp.`tpch/region.parquet` b " + "" +
" on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter
- " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') " + // left filter
- " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') " + // right filter
- " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
+ " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') "); // left filter
+ // " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') " + // right filter
+ // " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
}
@Test
[41/47] git commit: DRILL-1313: All text mode for json reader
Posted by ja...@apache.org.
DRILL-1313: All text mode for json reader
Current implementation handles nulls that appear while in text mode differently depending if they appear in lists or maps. This allows for a null where a list or map is expected to act the same way it does without text mode enabled. For an expected map it just assumes that the field didn't exist, in which case the leaves below become null filled, and for a list it will default to showing an empty list.
If we are actually inside of a list, a null in JSON will be treated the same as the string "null", which improves over the previous behavior of just dropping the null value all together, as we do not support null values within any of the repeated primitive vectors currently.
Patch has been rebased on top of merge branch.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/528308c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/528308c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/528308c3
Branch: refs/heads/master
Commit: 528308c325efd7804572e6d34819d8b3da7ff3b4
Parents: 929d765
Author: Jason Altekruse <al...@gmail.com>
Authored: Mon Aug 18 16:57:16 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 18:38:32 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/ExecConstants.java | 3 +-
.../exec/expr/fn/impl/conv/JsonConvertFrom.java | 4 +-
.../server/options/SystemOptionManager.java | 1 +
.../exec/store/easy/json/JSONRecordReader2.java | 4 +-
.../drill/exec/vector/AllocationHelper.java | 11 ++-
.../exec/vector/complex/RepeatedMapVector.java | 2 +-
.../exec/vector/complex/fn/JsonReader.java | 76 +++++++++++++++++---
.../vector/complex/fn/JsonReaderWithState.java | 6 +-
.../vector/complex/writer/TestJsonReader.java | 30 +++++++-
.../store/json/null_where_list_expected.json | 9 +++
.../store/json/null_where_map_expected.json | 9 +++
.../store/json/schema_change_int_to_string.json | 6 +-
.../drill/jdbc/test/TestJdbcDistQuery.java | 5 ++
13 files changed, 140 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index f018b78..77b51eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -85,7 +85,8 @@ public interface ExecConstants {
public static String PARQUET_NEW_RECORD_READER = "store.parquet.use_new_reader";
public static OptionValidator PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR = new BooleanValidator(PARQUET_NEW_RECORD_READER, false);
-
+ public static String JSON_ALL_TEXT_MODE = "store.json.all_text_mode";
+ public static OptionValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE, false);
public static final String SLICE_TARGET = "planner.slice_target";
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
index e66cabb..72bec3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java
@@ -67,7 +67,7 @@ public class JsonConvertFrom {
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
try {
- org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer, null);
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer, null, false);
jsonReader.write(new java.io.StringReader(input), writer);
@@ -94,7 +94,7 @@ public class JsonConvertFrom {
String input = new String(buf, com.google.common.base.Charsets.UTF_8);
try {
- org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer, null);
+ org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader(buffer, null, false);
jsonReader.write(new java.io.StringReader(input), writer);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index b322086..d4ff627 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -61,6 +61,7 @@ public class SystemOptionManager implements OptionManager{
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR,
+ ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR,
ExecConstants.SLICE_TARGET_OPTION,
ExecConstants.AFFINITY_FACTOR,
ExecConstants.MAX_WIDTH_GLOBAL,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
index 5bfc482..ee78c39 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
@@ -58,6 +58,7 @@ public class JSONRecordReader2 implements RecordReader{
private FragmentContext fragmentContext;
private OperatorContext operatorContext;
private List<SchemaPath> columns;
+ private boolean enableAllTextMode;
public JSONRecordReader2(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem,
List<SchemaPath> columns) throws OutOfMemoryException {
@@ -65,6 +66,7 @@ public class JSONRecordReader2 implements RecordReader{
this.fileSystem = fileSystem;
this.fragmentContext = fragmentContext;
this.columns = columns;
+ enableAllTextMode = fragmentContext.getDrillbitContext().getOptionManager().getOption("store.json.all_text_mode").bool_val;
}
@Override
@@ -74,7 +76,7 @@ public class JSONRecordReader2 implements RecordReader{
JsonRecordSplitter splitter = new UTF8JsonRecordSplitter(stream);
this.writer = new VectorContainerWriter(output);
this.mutator = output;
- jsonReader = new JsonReaderWithState(splitter, fragmentContext.getManagedBuffer(), columns);
+ jsonReader = new JsonReaderWithState(splitter, fragmentContext.getManagedBuffer(), columns, enableAllTextMode);
}catch(Exception e){
handleAndRaise("Failure reading JSON file.", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
index f7a74c2..51726a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
@@ -24,17 +24,22 @@ public class AllocationHelper {
allocate(v, valueCount, bytesPerValue, 5);
}
- public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+ public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
if(v instanceof FixedWidthVector){
((FixedWidthVector) v).allocateNew(valueCount);
} else if (v instanceof VariableWidthVector) {
((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
}else if(v instanceof RepeatedFixedWidthVector){
- ((RepeatedFixedWidthVector) v).allocateNew(valueCount, valueCount * repeatedPerTop);
+ ((RepeatedFixedWidthVector) v).allocateNew(valueCount, childValCount);
}else if(v instanceof RepeatedVariableWidthVector){
- ((RepeatedVariableWidthVector) v).allocateNew(valueCount * bytesPerValue * repeatedPerTop, valueCount, valueCount * repeatedPerTop);
+ ((RepeatedVariableWidthVector) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
}else{
v.allocateNew();
}
}
+
+ public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+ allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index f7baaa9..3fd1c12 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -81,7 +81,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
offsets.allocateNew(parentValueCount+1);
offsets.zeroVector();
for(ValueVector v : vectors.values()){
- AllocationHelper.allocate(v, parentValueCount, 50, childValueCount);
+ AllocationHelper.allocatePrecomputedChildCount(v, parentValueCount, 50, childValueCount);
}
mutator.reset();
accessor.reset();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 8393dc6..79c94c8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -24,6 +24,7 @@ import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.expr.holders.BigIntHolder;
@@ -62,12 +63,13 @@ public class JsonReader {
// A flag set at setup time if the start column is in the requested column list, prevents
// doing a more computational intensive check if we are supposed to be reading a column
private boolean starRequested;
+ private boolean allTextMode;
public JsonReader() throws IOException {
- this(null, null);
+ this(null, null, false);
}
- public JsonReader(DrillBuf managedBuf, List<SchemaPath> columns) throws JsonParseException, IOException {
+ public JsonReader(DrillBuf managedBuf, List<SchemaPath> columns, boolean allTextMode) throws JsonParseException, IOException {
factory.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
factory.configure(Feature.ALLOW_COMMENTS, true);
this.workBuf = managedBuf;
@@ -76,6 +78,7 @@ public class JsonReader {
if (this.columns == null) {
this.columns = new ArrayList();
this.columns.add(new SchemaPath(new PathSegment.NameSegment("*")));
+ this.allTextMode = allTextMode;
}
this.columnsFound = new boolean[this.columns.size()];
this.starRequested = containsStar();
@@ -201,34 +204,53 @@ public class JsonReader {
case VALUE_EMBEDDED_OBJECT:
case VALUE_FALSE: {
+ if (allTextMode) {
+ handleString(parser, map, fieldName);
+ break;
+ }
BitHolder h = new BitHolder();
h.value = 0;
map.bit(fieldName).write(h);
break;
}
case VALUE_TRUE: {
+ if (allTextMode) {
+ handleString(parser, map, fieldName);
+ break;
+ }
BitHolder h = new BitHolder();
h.value = 1;
map.bit(fieldName).write(h);
break;
}
case VALUE_NULL:
+ if (allTextMode) {
+ map.checkValueCapacity();
+ break;
+ }
map.checkValueCapacity();
// do nothing as we don't have a type.
break;
case VALUE_NUMBER_FLOAT:
+ if (allTextMode) {
+ handleString(parser, map, fieldName);
+ break;
+ }
Float8Holder fh = new Float8Holder();
fh.value = parser.getDoubleValue();
map.float8(fieldName).write(fh);
break;
case VALUE_NUMBER_INT:
+ if (allTextMode) {
+ handleString(parser, map, fieldName);
+ break;
+ }
BigIntHolder bh = new BigIntHolder();
bh.value = parser.getLongValue();
map.bigInt(fieldName).write(bh);
break;
case VALUE_STRING:
- VarCharHolder vh = new VarCharHolder();
- map.varChar(fieldName).write(prepareVarCharHolder(vh, parser));
+ handleString(parser, map, fieldName);
break;
default:
@@ -245,8 +267,7 @@ public class JsonReader {
workBuf = workBuf.reallocIfNeeded(length);
}
- private VarCharHolder prepareVarCharHolder(VarCharHolder vh, JsonParser parser) throws IOException {
- String value = parser.getText();
+ private VarCharHolder prepareVarCharHolder(VarCharHolder vh, String value) throws IOException {
byte[] b = value.getBytes(Charsets.UTF_8);
ensure(b.length);
workBuf.setBytes(0, b);
@@ -256,6 +277,21 @@ public class JsonReader {
return vh;
}
+ private void handleString(JsonParser parser, MapWriter writer, String fieldName) throws IOException {
+ VarCharHolder vh = new VarCharHolder();
+ writer.varChar(fieldName).write(prepareVarCharHolder(vh, parser.getText()));
+ }
+
+ private void handleString(JsonParser parser, ListWriter writer) throws IOException {
+ VarCharHolder vh = new VarCharHolder();
+ writer.varChar().write(prepareVarCharHolder(vh, parser.getText()));
+ }
+
+ private void handleString(String value, ListWriter writer) throws IOException {
+ VarCharHolder vh = new VarCharHolder();
+ writer.varChar().write(prepareVarCharHolder(vh, parser.getText()));
+ }
+
private void writeData(ListWriter list) throws JsonParseException, IOException {
list.start();
outside: while(true){
@@ -273,33 +309,53 @@ public class JsonReader {
case VALUE_EMBEDDED_OBJECT:
case VALUE_FALSE:{
+ if (allTextMode) {
+ handleString(parser, list);
+ break;
+ }
BitHolder h = new BitHolder();
h.value = 0;
list.bit().write(h);
break;
}
case VALUE_TRUE: {
+ if (allTextMode) {
+ handleString(parser, list);
+ break;
+ }
BitHolder h = new BitHolder();
h.value = 1;
list.bit().write(h);
break;
}
case VALUE_NULL:
- // do nothing as we don't have a type.
- break;
+ if (allTextMode) {
+ handleString("null", list);
+ break;
+ }
+ throw new DrillRuntimeException("Null values are not supported in lists be default. " +
+ "Please set jason_all_text_mode to true to read lists containing nulls. " +
+ "Be advised that this will treat JSON null values as string containing the word 'null'.");
case VALUE_NUMBER_FLOAT:
+ if (allTextMode) {
+ handleString(parser, list);
+ break;
+ }
Float8Holder fh = new Float8Holder();
fh.value = parser.getDoubleValue();
list.float8().write(fh);
break;
case VALUE_NUMBER_INT:
+ if (allTextMode) {
+ handleString(parser, list);
+ break;
+ }
BigIntHolder bh = new BigIntHolder();
bh.value = parser.getLongValue();
list.bigInt().write(bh);
break;
case VALUE_STRING:
- VarCharHolder vh = new VarCharHolder();
- list.varChar().write(prepareVarCharHolder(vh, parser));
+ handleString(parser, list);
break;
default:
throw new IllegalStateException("Unexpected token " + parser.getCurrentToken());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
index 13b8215..c2dcc95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
@@ -39,14 +39,14 @@ public class JsonReaderWithState {
private JsonRecordSplitter splitter;
private JsonReader jsonReader;
- public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace, List<SchemaPath> columns) throws IOException{
+ public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace, List<SchemaPath> columns, boolean allTextMode) throws IOException{
this.splitter = splitter;
reader = splitter.getNextReader();
- jsonReader = new JsonReader(workspace, columns);
+ jsonReader = new JsonReader(workspace, columns, allTextMode);
}
public JsonReaderWithState(JsonRecordSplitter splitter) throws IOException{
- this(splitter, null, null);
+ this(splitter, null, null, false);
}
public List<SchemaPath> getNullColumns() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 885e50a..bfc0d20 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -96,7 +96,7 @@ public class TestJsonReader extends BaseTestQuery {
System.out.println("======");
}
int rowCount = testRunAndPrint(queryType, query);
- assertEquals( rowCount, rowCounts[i]);
+ assertEquals(rowCounts[i], rowCount);
System.out.println();
i++;
}
@@ -118,6 +118,14 @@ public class TestJsonReader extends BaseTestQuery {
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
}
+ public void testAllTextMode() throws Exception {
+ test("alter system set `store.json.all_text_mode` = true");
+ String[] queries = {"select * from cp.`/store/json/schema_change_int_to_string.json`"};
+ long[] rowCounts = {3};
+ String filename = "/store/json/schema_change_int_to_string.json";
+ runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
+ }
+
@Test
public void readComplexWithStar() throws Exception {
List<QueryResultBatch> results = testSqlWithResults("select * from cp.`/store/json/test_complex_read_with_star.json`");
@@ -134,6 +142,24 @@ public class TestJsonReader extends BaseTestQuery {
batchLoader.clear();
}
+ @Test
+ public void testNullWhereListExpected() throws Exception {
+ test("alter system set `store.json.all_text_mode` = true");
+ String[] queries = {"select * from cp.`/store/json/null_where_list_expected.json`"};
+ long[] rowCounts = {3};
+ String filename = "/store/json/null_where_list_expected.json";
+ runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
+ }
+
+ @Test
+ public void testNullWhereMapExpected() throws Exception {
+ test("alter system set `store.json.all_text_mode` = true");
+ String[] queries = {"select * from cp.`/store/json/null_where_map_expected.json`"};
+ long[] rowCounts = {3};
+ String filename = "/store/json/null_where_map_expected.json";
+ runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
+ }
+
// The project pushdown rule is correctly adding the projected columns to the scan, however it is not removing
// the redundant project operator after the scan, this tests runs a physical plan generated from one of the tests to
// ensure that the project is filtering out the correct data in the scan alone
@@ -245,7 +271,7 @@ public class TestJsonReader extends BaseTestQuery {
writer.allocate();
DrillBuf buffer = allocator.buffer(255);
- JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound), buffer, null);
+ JsonReaderWithState jsonReader = new JsonReaderWithState(new ReaderJSONRecordSplitter(compound), buffer, null, false);
int i =0;
List<Integer> batchSizes = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/test/resources/store/json/null_where_list_expected.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/null_where_list_expected.json b/exec/java-exec/src/test/resources/store/json/null_where_list_expected.json
new file mode 100644
index 0000000..aca7ffc
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/null_where_list_expected.json
@@ -0,0 +1,9 @@
+{
+ "list_1" : [1,2,3]
+}
+{
+ "list_1" : null
+}
+{
+ "list_1" : [4,5,6]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/test/resources/store/json/null_where_map_expected.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/null_where_map_expected.json b/exec/java-exec/src/test/resources/store/json/null_where_map_expected.json
new file mode 100644
index 0000000..cd4ca2b
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/null_where_map_expected.json
@@ -0,0 +1,9 @@
+{
+ "map_1" : { "f_1" : 1, "f_2" : 2, "f_3" : 3}
+}
+{
+ "map_1" : null
+}
+{
+ "map_1" : { "f_1" : 3, "f_2" : 4, "f_3" : 5}
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json b/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
index f2fca86..0943e69 100644
--- a/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
+++ b/exec/java-exec/src/test/resources/store/json/schema_change_int_to_string.json
@@ -12,7 +12,7 @@
"inner_2" : 3,
"inner_3" : { "inner_object_field_1" : 2}
},
- "field_5" : [ { "inner_list" : [1,6] }, { "inner_list":[3,8]}, { "inner_list":[12,4,5]} ]
+ "field_5" : [ { "inner_list" : [1, null, 6] }, { "inner_list":[3,8]}, { "inner_list":[12, null, 4, "null", 5]} ]
}
{
"field_1": [5,10,15],
@@ -20,11 +20,11 @@
"field_3": {
"inner_1" : 5,
"inner_2" : 3,
- "inner_3" : [ { "inner_object_field_1" : 2}, {"inner_object_field_1" : 10} ]
+ "inner_3" : [ { "inner_object_field_1" : null}, {"inner_object_field_1" : 10} ]
},
"field_4" : {
"inner_1" : [4,5,6],
"inner_2" : 3
},
- "field_5" : [ { "inner_list" : [5,6.0, "1234"] }, { "inner_list":[7,8.0, "12341324"], "inner_list_2" : [1,2,2323.443e10, "hello there"]}, { "inner_list":[3,4,5], "inner_list_2" : [10, 11, 12]} ]
+ "field_5" : [ { "inner_list" : [5, null, 6.0, "1234"] }, { "inner_list":[7,8.0, "12341324"], "inner_list_2" : [1,2,2323.443e10, "hello there"]}, { "inner_list":[3,4,5], "inner_list_2" : [10, 11, 12]} ]
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/528308c3/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
index 39ba043..d10eeb2 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
@@ -35,6 +35,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.Ignore;
import org.junit.rules.TestRule;
import com.google.common.base.Stopwatch;
@@ -110,6 +111,7 @@ public class TestJdbcDistQuery extends JdbcTest{
+ "ORDER BY R_REGIONKEY", WORKING_PATH));
}
+ @Ignore
@Test
public void testJoinSingleFile() throws Exception{
testQuery(String.format("select T1.R_REGIONKEY "
@@ -118,6 +120,7 @@ public class TestJdbcDistQuery extends JdbcTest{
+ "on T1.R_REGIONKEY = T2.N_REGIONKEY", WORKING_PATH, WORKING_PATH));
}
+ @Ignore
@Test
public void testJoinMultiFile() throws Exception{
testQuery(String.format("select T1.R_REGIONKEY "
@@ -126,6 +129,7 @@ public class TestJdbcDistQuery extends JdbcTest{
+ "on T1.R_REGIONKEY = T2.N_REGIONKEY", WORKING_PATH, WORKING_PATH));
}
+ @Ignore
@Test
public void testJoinMFileWhere() throws Exception{
testQuery(String.format("select T1.R_REGIONKEY, T1.R_NAME "
@@ -159,6 +163,7 @@ public class TestJdbcDistQuery extends JdbcTest{
+ "order by R_REGIONKEY ", WORKING_PATH ));
}
+ @Ignore
@Test
public void testJoinAggSortWhere() throws Exception{
testQuery(String.format("select T1.R_REGIONKEY, COUNT(1) as CNT "
[18/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/IndexPointer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/IndexPointer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/IndexPointer.java
new file mode 100644
index 0000000..020c5d8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/IndexPointer.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+public class IndexPointer {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IndexPointer.class);
+
+ public int value;
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 7233f69..15044b8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.physical.impl.join;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import org.apache.drill.common.expression.FieldReference;
@@ -35,7 +34,6 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.holders.IntHolder;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
@@ -44,8 +42,8 @@ import org.apache.drill.exec.physical.impl.common.ChainedHashTable;
import org.apache.drill.exec.physical.impl.common.HashTable;
import org.apache.drill.exec.physical.impl.common.HashTableConfig;
import org.apache.drill.exec.physical.impl.common.HashTableStats;
+import org.apache.drill.exec.physical.impl.common.IndexPointer;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
-import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch;
import org.apache.drill.exec.record.AbstractRecordBatch;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.ExpandableHyperContainer;
@@ -54,7 +52,6 @@ import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.allocator.VectorAllocator;
import org.eigenbase.rel.JoinRelType;
import com.sun.codemodel.JExpr;
@@ -155,7 +152,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
NUM_ENTRIES,
NUM_RESIZING,
RESIZING_TIME;
-
+
// duplicate for hash ag
@Override
@@ -163,7 +160,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
return ordinal();
}
}
-
+
@Override
public int getRecordCount() {
return outputRecords;
@@ -339,7 +336,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
hjHelper.addNewBatch(currentRecordCount);
// Holder contains the global index where the key is hashed into using the hash table
- IntHolder htIndex = new IntHolder();
+ IndexPointer htIndex = new IndexPointer();
// For every record in the build batch , hash the key columns
for (int i = 0; i < currentRecordCount; i++) {
@@ -491,7 +488,9 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
@Override
public void cleanup() {
- hjHelper.clear();
+ if(hjHelper != null){
+ hjHelper.clear();
+ }
// If we didn't receive any data, hyperContainer may be null, check before clearing
if (hyperContainer != null) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index c5d342f..994eea4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -219,6 +219,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
phyRelNode = ProducerConsumerPrelVisitor.addProducerConsumerToScans(phyRelNode, (int) queueSize);
}
+
/* 6.)
* if the client does not support complex types (Map, Repeated)
* insert a project which which would convert
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
index 61c1689..ac26969 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/DeadBuf.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.record;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufProcessor;
+import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.io.InputStream;
@@ -32,14 +33,14 @@ import java.nio.charset.Charset;
public class DeadBuf extends ByteBuf {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DeadBuf.class);
-
+
private static final String ERROR_MESSAGE = "Attemped to access a DeadBuf. This would happen if you attempted to interact with a buffer that has been moved or not yet initialized.";
-
- public static final DeadBuf DEAD_BUFFER = new DeadBuf();
+
+ public static final DrillBuf DEAD_BUFFER = null;
private DeadBuf(){}
-
+
@Override
public boolean isReadable(int size) {
throw new UnsupportedOperationException(ERROR_MESSAGE);
@@ -654,7 +655,7 @@ public class DeadBuf extends ByteBuf {
}
-
+
@Override
public int bytesBefore(byte value) {
throw new UnsupportedOperationException(ERROR_MESSAGE);
@@ -667,14 +668,14 @@ public class DeadBuf extends ByteBuf {
}
-
+
@Override
public int bytesBefore(int index, int length, byte value) {
throw new UnsupportedOperationException(ERROR_MESSAGE);
}
-
+
@Override
public ByteBuf copy() {
throw new UnsupportedOperationException(ERROR_MESSAGE);
@@ -839,6 +840,6 @@ public class DeadBuf extends ByteBuf {
public String toString() {
return null;
}
-
-
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
index 0cca9cd..dbf17c4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.record;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
@@ -29,10 +30,10 @@ public class RawFragmentBatch {
final RemoteConnection connection;
final FragmentRecordBatch header;
- final ByteBuf body;
+ final DrillBuf body;
final ResponseSender sender;
- public RawFragmentBatch(RemoteConnection connection, FragmentRecordBatch header, ByteBuf body, ResponseSender sender) {
+ public RawFragmentBatch(RemoteConnection connection, FragmentRecordBatch header, DrillBuf body, ResponseSender sender) {
super();
this.header = header;
this.body = body;
@@ -45,7 +46,7 @@ public class RawFragmentBatch {
return header;
}
- public ByteBuf getBody() {
+ public DrillBuf getBody() {
return body;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index 0caed7d..d756685 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -17,13 +17,12 @@
*/
package org.apache.drill.exec.record;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import io.netty.buffer.EmptyByteBuf;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
@@ -57,7 +56,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp
* @return Whether or not the schema changed since the previous load.
* @throws SchemaChangeException
*/
- public boolean load(RecordBatchDef def, ByteBuf buf) throws SchemaChangeException {
+ public boolean load(RecordBatchDef def, DrillBuf buf) throws SchemaChangeException {
// logger.debug("Loading record batch with def {} and data {}", def, buf);
container.zeroVectors();
this.valueCount = def.getRecordCount();
@@ -85,7 +84,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp
}
if (fmd.getValueCount() == 0 && (!fmd.hasGroupCount() || fmd.getGroupCount() == 0)) {
// v.clear();
- v.load(fmd, new EmptyByteBuf(allocator.getUnderlyingAllocator()));
+ v.load(fmd, allocator.buffer(0));
} else {
v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength()));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
index 22aa731..3154e6e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
@@ -231,5 +231,14 @@ public class TypedFieldId {
return true;
}
+ @Override
+ public String toString() {
+ final int maxLen = 10;
+ return "TypedFieldId [fieldIds="
+ + (fieldIds != null ? Arrays.toString(Arrays.copyOf(fieldIds, Math.min(fieldIds.length, maxLen))) : null)
+ + ", remainder=" + remainder + "]";
+ }
+
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
index b55ed82..3810115 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
@@ -17,8 +17,7 @@
*/
package org.apache.drill.exec.record;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.List;
@@ -37,15 +36,15 @@ public class WritableBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WritableBatch.class);
private final RecordBatchDef def;
- private final ByteBuf[] buffers;
+ private final DrillBuf[] buffers;
private boolean cleared = false;
- private WritableBatch(RecordBatchDef def, List<ByteBuf> buffers) {
+ private WritableBatch(RecordBatchDef def, List<DrillBuf> buffers) {
this.def = def;
- this.buffers = buffers.toArray(new ByteBuf[buffers.size()]);
+ this.buffers = buffers.toArray(new DrillBuf[buffers.size()]);
}
- private WritableBatch(RecordBatchDef def, ByteBuf[] buffers) {
+ private WritableBatch(RecordBatchDef def, DrillBuf[] buffers) {
super();
this.def = def;
this.buffers = buffers;
@@ -55,20 +54,27 @@ public class WritableBatch {
return def;
}
- public ByteBuf[] getBuffers() {
+ public DrillBuf[] getBuffers() {
return buffers;
}
public void reconstructContainer(VectorContainer container) {
Preconditions.checkState(!cleared,
"Attempted to reconstruct a container from a WritableBatch after it had been cleared");
- if (buffers.length > 0) { /* If we have ByteBuf's associated with value vectors */
+ if (buffers.length > 0) { /* If we have DrillBuf's associated with value vectors */
+ int len = 0;
+ for(DrillBuf b : buffers){
+ len += b.capacity();
+ }
- CompositeByteBuf cbb = new CompositeByteBuf(buffers[0].alloc(), true, buffers.length);
+ DrillBuf newBuf = buffers[0].getAllocator().buffer(len);
/* Copy data from each buffer into the compound buffer */
- for (ByteBuf buf : buffers) {
- cbb.addComponent(buf);
+ int offset = 0;
+ for (DrillBuf buf : buffers) {
+ newBuf.setBytes(offset, buf);
+ offset += buf.capacity();
+ buf.release();
}
List<SerializedField> fields = def.getFieldList();
@@ -83,7 +89,7 @@ public class WritableBatch {
for (VectorWrapper<?> vv : container) {
SerializedField fmd = fields.get(vectorIndex);
ValueVector v = vv.getValueVector();
- ByteBuf bb = cbb.slice(bufferOffset, fmd.getBufferLength());
+ DrillBuf bb = newBuf.slice(bufferOffset, fmd.getBufferLength());
// v.load(fmd, cbb.slice(bufferOffset, fmd.getBufferLength()));
v.load(fmd, bb);
bb.release();
@@ -109,7 +115,7 @@ public class WritableBatch {
public void clear() {
if(cleared) return;
- for (ByteBuf buf : buffers) {
+ for (DrillBuf buf : buffers) {
buf.release();
}
cleared = true;
@@ -125,7 +131,7 @@ public class WritableBatch {
}
public static WritableBatch getBatchNoHV(int recordCount, Iterable<ValueVector> vectors, boolean isSV2) {
- List<ByteBuf> buffers = Lists.newArrayList();
+ List<DrillBuf> buffers = Lists.newArrayList();
List<SerializedField> metadata = Lists.newArrayList();
for (ValueVector vv : vectors) {
@@ -137,7 +143,7 @@ public class WritableBatch {
continue;
}
- for (ByteBuf b : vv.getBuffers()) {
+ for (DrillBuf b : vv.getBuffers()) {
buffers.add(b);
}
// remove vv access to buffers.
@@ -159,13 +165,13 @@ public class WritableBatch {
}
public void retainBuffers() {
- for (ByteBuf buf : buffers) {
+ for (DrillBuf buf : buffers) {
buf.retain();
}
}
-
+
public void retainBuffers(int increment) {
- for (ByteBuf buf : buffers) {
+ for (DrillBuf buf : buffers) {
buf.retain(increment);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
index af1e2b6..038bb2f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.record.selection;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.io.Closeable;
import java.io.IOException;
@@ -33,7 +33,7 @@ public class SelectionVector2 implements Closeable{
private final BufferAllocator allocator;
private int recordCount;
- private ByteBuf buffer = DeadBuf.DEAD_BUFFER;
+ private DrillBuf buffer = DeadBuf.DEAD_BUFFER;
public static final int RECORD_SIZE = 2;
@@ -45,9 +45,9 @@ public class SelectionVector2 implements Closeable{
return recordCount;
}
- public ByteBuf getBuffer()
+ public DrillBuf getBuffer()
{
- ByteBuf bufferHandle = this.buffer;
+ DrillBuf bufferHandle = this.buffer;
/* Increment the ref count for this buffer */
bufferHandle.retain();
@@ -60,7 +60,7 @@ public class SelectionVector2 implements Closeable{
return bufferHandle;
}
- public void setBuffer(ByteBuf bufferHandle)
+ public void setBuffer(DrillBuf bufferHandle)
{
/* clear the existing buffer */
clear();
@@ -70,7 +70,6 @@ public class SelectionVector2 implements Closeable{
}
public char getIndex(int index){
-
return buffer.getChar(index * RECORD_SIZE);
}
@@ -78,6 +77,14 @@ public class SelectionVector2 implements Closeable{
buffer.setChar(index * RECORD_SIZE, value);
}
+ public long getDataAddr(){
+ return buffer.memoryAddress();
+ }
+
+ public void setIndex(int index, int value){
+ buffer.setChar(index, value);
+ }
+
public boolean allocateNew(int size){
clear();
buffer = allocator.buffer(size * RECORD_SIZE);
@@ -100,7 +107,7 @@ public class SelectionVector2 implements Closeable{
clear();
return newSV;
}
-
+
public void clear() {
if (buffer != null && buffer != DeadBuf.DEAD_BUFFER) {
buffer.release();
@@ -108,7 +115,7 @@ public class SelectionVector2 implements Closeable{
recordCount = 0;
}
}
-
+
public void setRecordCount(int recordCount){
// logger.debug("Seting record count to {}", recordCount);
this.recordCount = recordCount;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index 8176567..93f5b6d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -30,9 +30,9 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
+import org.apache.drill.exec.util.DecimalUtility;
public class TypeCastRules {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
index 473e3e6..02fb75e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
@@ -18,8 +18,6 @@
package org.apache.drill.exec.rpc;
import io.netty.buffer.ByteBuf;
-import io.netty.buffer.SwappedByteBuf;
-import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.codec.CorruptedFrameException;
@@ -29,7 +27,6 @@ import java.util.List;
import org.apache.drill.exec.memory.BufferAllocator;
import com.google.protobuf.CodedInputStream;
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
/**
* Modified version of ProtobufVarint32FrameDecoder that avoids bytebuf copy.
@@ -39,7 +36,7 @@ public class ProtobufLengthDecoder extends ByteToMessageDecoder {
private BufferAllocator allocator;
private OutOfMemoryHandler outOfMemoryHandler;
-
+
public ProtobufLengthDecoder(BufferAllocator allocator, OutOfMemoryHandler outOfMemoryHandler) {
super();
this.allocator = allocator;
@@ -90,14 +87,14 @@ public class ProtobufLengthDecoder extends ByteToMessageDecoder {
return;
}
outBuf.writeBytes(in, in.readerIndex(), length);
-
+
in.skipBytes(length);
-
+
if (RpcConstants.EXTRA_DEBUGGING)
logger.debug(String.format(
"ReaderIndex is %d after length header of %d bytes and frame body of length %d bytes.",
in.readerIndex(), i + 1, length));
-
+
out.add(outBuf);
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
index f1c94d3..d4a73c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
@@ -17,11 +17,10 @@
*/
package org.apache.drill.exec.rpc.data;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
import org.apache.drill.exec.rpc.RemoteConnection;
-import org.apache.drill.exec.rpc.Response;
import org.apache.drill.exec.rpc.ResponseSender;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.work.fragment.FragmentManager;
@@ -29,7 +28,7 @@ import org.apache.drill.exec.work.fragment.FragmentManager;
public interface DataResponseHandler {
public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch,
- ByteBuf data, ResponseSender responder) throws RpcException;
+ DrillBuf data, ResponseSender responder) throws RpcException;
public void informOutOfMemory();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
index 5d71c90..c37550f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.rpc.data;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
@@ -48,7 +48,7 @@ public class DataResponseHandlerImpl implements DataResponseHandler{
}
- public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, ByteBuf data, ResponseSender sender) throws RpcException {
+ public void handle(RemoteConnection connection, FragmentManager manager, FragmentRecordBatch fragmentBatch, DrillBuf data, ResponseSender sender) throws RpcException {
// logger.debug("Fragment Batch received {}", fragmentBatch);
try {
boolean canRun = manager.handle(new RawFragmentBatch(connection, fragmentBatch, data, sender));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
index 42dee94..8611b5a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.rpc.data;
-import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
@@ -114,11 +114,11 @@ public class DataServer extends BasicServer<RpcType, BitServerConnection> {
}
BufferAllocator allocator = manager.getFragmentContext().getAllocator();
if(body != null){
- if(!allocator.takeOwnership((AccountingByteBuf) body.unwrap())){
+ if(!allocator.takeOwnership((DrillBuf) body.unwrap())){
dataHandler.handle(connection, manager, OOM_FRAGMENT, null, null);
}
}
- dataHandler.handle(connection, manager, fragmentBatch, body, sender);
+ dataHandler.handle(connection, manager, fragmentBatch, (DrillBuf) body, sender);
} catch (FragmentSetupException e) {
logger.error("Failure while getting fragment manager. {}", QueryIdHelper.getQueryIdentifier(handle), e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultBatch.java
index 98e5943..e36a1c6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultBatch.java
@@ -17,32 +17,33 @@
*/
package org.apache.drill.exec.rpc.user;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.proto.UserBitShared.QueryResult;
public class QueryResultBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryResultBatch.class);
-
+
private final QueryResult header;
- private final ByteBuf data;
-
- public QueryResultBatch(QueryResult header, ByteBuf data) {
+ private final DrillBuf data;
+
+ public QueryResultBatch(QueryResult header, DrillBuf data) {
// logger.debug("New Result Batch with header {} and data {}", header, data);
this.header = header;
this.data = data;
if(this.data != null) data.retain();
}
+
public QueryResult getHeader() {
return header;
}
- public ByteBuf getData() {
+ public DrillBuf getData() {
return data;
}
-
-
+
+
public boolean hasData(){
return data != null;
}
@@ -50,12 +51,12 @@ public class QueryResultBatch {
public void release(){
if(data != null) data.release();
}
-
+
@Override
public String toString() {
return "QueryResultBatch [header=" + header + ", data=" + data + "]";
}
-
-
-
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
index 95b7446..4bbb13e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.rpc.user;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
@@ -55,7 +56,7 @@ public class QueryResultHandler {
public void batchArrived(ConnectionThrottle throttle, ByteBuf pBody, ByteBuf dBody) throws RpcException {
final QueryResult result = RpcBus.get(pBody, QueryResult.PARSER);
- final QueryResultBatch batch = new QueryResultBatch(result, dBody);
+ final QueryResultBatch batch = new QueryResultBatch(result, (DrillBuf) dBody);
final boolean failed = (batch.getHeader().getQueryState() == QueryState.FAILED);
assert failed || batch.getHeader().getErrorCount() == 0 : "Error count for the query batch is non-zero but QueryState != FAILED";
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index eed0126..7d48711 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.cache.DistributedCache;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -57,6 +58,7 @@ public class DrillbitContext {
private final FunctionImplementationRegistry functionRegistry;
private final SystemOptionManager systemOptions;
private final PStoreProvider provider;
+ private final CodeCompiler compiler;
public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus, PStoreProvider provider) {
super();
@@ -77,8 +79,7 @@ public class DrillbitContext {
this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
this.systemOptions = new SystemOptionManager(context.getConfig(), provider);
-
-// this.globalDrillOptions = new DistributedGlobalOptions(this.cache);
+ this.compiler = new CodeCompiler(context.getConfig(), cache, systemOptions);
}
public FunctionImplementationRegistry getFunctionImplementationRegistry() {
@@ -153,4 +154,10 @@ public class DrillbitContext {
return coord;
}
+ public CodeCompiler getCompiler() {
+ return compiler;
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index eddb818..b322086 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -99,9 +99,10 @@ public class SystemOptionManager implements OptionManager{
.build();
}
- public void init() throws IOException{
+ public SystemOptionManager init() throws IOException{
this.options = provider.getPStore(config);
this.admin = new SystemOptionAdmin();
+ return this;
}
private class Iter implements Iterator<OptionValue>{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
index cd3cfdc..5d741e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
@@ -59,7 +59,7 @@ public class JSONRecordReader2 implements RecordReader{
List<SchemaPath> columns) throws OutOfMemoryException {
this.hadoopPath = new Path(inputPath);
this.fileSystem = fileSystem;
- this.fragmentContext=fragmentContext;
+ this.fragmentContext = fragmentContext;
}
@Override
@@ -69,9 +69,9 @@ public class JSONRecordReader2 implements RecordReader{
JsonRecordSplitter splitter = new UTF8JsonRecordSplitter(stream);
this.writer = new VectorContainerWriter(output);
this.mutator = output;
- jsonReader = new JsonReaderWithState(splitter);
+ jsonReader = new JsonReaderWithState(splitter, fragmentContext.getManagedBuffer());
}catch(Exception e){
- handleAndRaise("Failure reading JSON file.", e);
+ throw new ExecutionSetupException("Failure reading JSON file.", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
index 5c25e11..54771e4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
@@ -18,16 +18,19 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
import org.apache.drill.exec.store.ParquetOutputRecordWriter;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.DateVector;
import org.apache.drill.exec.vector.Decimal28SparseVector;
import org.apache.drill.exec.vector.Decimal38SparseVector;
import org.apache.drill.exec.vector.ValueVector;
import org.joda.time.DateTimeUtils;
+
import parquet.column.ColumnDescriptor;
import parquet.format.SchemaElement;
import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -36,9 +39,9 @@ import java.math.BigDecimal;
class FixedByteAlignedReader extends ColumnReader {
- protected ByteBuf bytebuf;
+ protected DrillBuf bytebuf;
+
-
FixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
boolean fixedLength, ValueVector v, SchemaElement schemaElement) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
@@ -120,7 +123,7 @@ class FixedByteAlignedReader extends ColumnReader {
@Override
void addNext(int start, int index) {
int width = Decimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal28Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), Decimal28SparseHolder.nDecimalDigits);
}
@@ -139,7 +142,7 @@ class FixedByteAlignedReader extends ColumnReader {
@Override
void addNext(int start, int index) {
int width = Decimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal38Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), Decimal38SparseHolder.nDecimalDigits);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
index 4d7f312..f88d56a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
@@ -18,8 +18,10 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.NullableBigIntVector;
import org.apache.drill.exec.vector.NullableFloat4Vector;
import org.apache.drill.exec.vector.NullableFloat8Vector;
@@ -31,8 +33,8 @@ import org.apache.drill.exec.store.ParquetOutputRecordWriter;
import org.apache.drill.exec.vector.NullableDateVector;
import org.apache.drill.exec.vector.NullableDecimal28SparseVector;
import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
-
import org.joda.time.DateTimeUtils;
+
import parquet.column.ColumnDescriptor;
import parquet.format.SchemaElement;
import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -42,7 +44,7 @@ import java.math.BigDecimal;
public class NullableFixedByteAlignedReaders {
static class NullableFixedByteAlignedReader extends NullableColumnReader {
- protected ByteBuf bytebuf;
+ protected DrillBuf bytebuf;
NullableFixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v, SchemaElement schemaElement) throws ExecutionSetupException {
@@ -67,8 +69,6 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryIntReader extends NullableColumnReader<NullableIntVector> {
- private ByteBuf bytebuf;
-
NullableDictionaryIntReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableIntVector v,
SchemaElement schemaElement) throws ExecutionSetupException {
@@ -88,8 +88,6 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryBigIntReader extends NullableColumnReader<NullableBigIntVector> {
- private ByteBuf bytebuf;
-
NullableDictionaryBigIntReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableBigIntVector v,
SchemaElement schemaElement) throws ExecutionSetupException {
@@ -107,8 +105,6 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryFloat4Reader extends NullableColumnReader<NullableFloat4Vector> {
- private ByteBuf bytebuf;
-
NullableDictionaryFloat4Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableFloat4Vector v,
SchemaElement schemaElement) throws ExecutionSetupException {
@@ -126,8 +122,6 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryFloat8Reader extends NullableColumnReader<NullableFloat8Vector> {
- private ByteBuf bytebuf;
-
NullableDictionaryFloat8Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableFloat8Vector v,
SchemaElement schemaElement) throws ExecutionSetupException {
@@ -211,7 +205,7 @@ public class NullableFixedByteAlignedReaders {
@Override
void addNext(int start, int index) {
int width = NullableDecimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal28Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), NullableDecimal28SparseHolder.nDecimalDigits);
}
@@ -230,7 +224,7 @@ public class NullableFixedByteAlignedReaders {
@Override
void addNext(int start, int index) {
int width = NullableDecimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal38Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), NullableDecimal38SparseHolder.nDecimalDigits);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
index 83812ea..ba9ff80 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
@@ -18,8 +18,11 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.ValueVector;
+
import parquet.bytes.BytesUtils;
import parquet.column.ColumnDescriptor;
import parquet.format.SchemaElement;
@@ -38,7 +41,7 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
- public abstract boolean setSafe(int index, ByteBuf value, int start, int length);
+ public abstract boolean setSafe(int index, DrillBuf value, int start, int length);
public abstract int capacity();
@@ -82,8 +85,7 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten
}
else {
// re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
- dataTypeLengthInBits = BytesUtils.readIntLittleEndian(pageReader.pageDataByteArray.nioBuffer(),
- (int) pageReader.readyToReadPosInBytes);
+ dataTypeLengthInBits = pageReader.pageDataByteArray.getInt((int) pageReader.readyToReadPosInBytes);
}
// I think this also needs to happen if it is null for the random access
if (! variableWidthVector.getMutator().setValueLengthSafe((int) valuesReadInCurrentPass + pageReader.valuesReadyToRead, dataTypeLengthInBits)) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
index 1687c3b..4165cbd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
@@ -22,7 +22,9 @@ import java.util.ArrayList;
import java.util.List;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.Unpooled;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.store.parquet.ColumnDataReader;
@@ -54,7 +56,7 @@ final class PageReader {
// store references to the pages that have been uncompressed, but not copied to ValueVectors yet
Page currentPage;
// buffer to store bytes of current page
- ByteBuf pageDataByteArray;
+ DrillBuf pageDataByteArray;
// for variable length data we need to keep track of our current position in the page data
// as the values and lengths are intermixed, making random access to the length data impossible
@@ -134,10 +136,10 @@ final class PageReader {
throw new ExecutionSetupException("Error opening or reading metadata for parquet file at location: "
+ path.getName(), e);
}
-
+
}
-
+
/**
* Grab the next page.
*
@@ -226,7 +228,7 @@ final class PageReader {
return false;
}
- pageDataByteArray = Unpooled.wrappedBuffer(currentPage.getBytes().toByteBuffer());
+ pageDataByteArray = DrillBuf.wrapByteBuffer(currentPage.getBytes().toByteBuffer());
readPosInBytes = 0;
if (parentColumnReader.getColumnDescriptor().getMaxRepetitionLevel() > 0) {
@@ -272,7 +274,7 @@ final class PageReader {
readyToReadPosInBytes = readPosInBytes;
return true;
}
-
+
public void clear(){
this.dataReader.clear();
// Free all memory, including fixed length types. (Data is being copied for all types not just var length types)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
index 555cb94..ecfa110 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
@@ -17,18 +17,20 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet.columnreaders;
-import java.math.BigDecimal;
-
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.Unpooled;
+
+import java.math.BigDecimal;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.util.DecimalUtility;
-import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
-import org.apache.drill.exec.expr.holders.VarBinaryHolder;
-import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.Decimal28SparseVector;
import org.apache.drill.exec.vector.Decimal38SparseVector;
import org.apache.drill.exec.vector.NullableDecimal28SparseVector;
@@ -37,6 +39,7 @@ import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.NullableVarCharVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
+
import parquet.column.ColumnDescriptor;
import parquet.format.SchemaElement;
import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -56,9 +59,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
+ public boolean setSafe(int index, DrillBuf bytebuf, int start, int length) {
int width = Decimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= decimal28Vector.getValueCapacity()) {
return false;
}
@@ -85,9 +88,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
+ public boolean setSafe(int index, DrillBuf bytebuf, int start, int length) {
int width = Decimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= nullableDecimal28Vector.getValueCapacity()) {
return false;
}
@@ -115,9 +118,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
+ public boolean setSafe(int index, DrillBuf bytebuf, int start, int length) {
int width = Decimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= decimal28Vector.getValueCapacity()) {
return false;
}
@@ -144,9 +147,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
+ public boolean setSafe(int index, DrillBuf bytebuf, int start, int length) {
int width = Decimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromDrillBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= nullableDecimal38Vector.getValueCapacity()) {
return false;
}
@@ -176,12 +179,12 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
+ public boolean setSafe(int index, DrillBuf bytebuf, int start, int length) {
boolean success;
if(index >= varCharVector.getValueCapacity()) return false;
if (usingDictionary) {
- ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
+ DrillBuf b = DrillBuf.wrapByteBuffer(currDictVal.toByteBuffer());
int st=0;
int len=currDictVal.length();
VarCharHolder holder = new VarCharHolder();
@@ -211,44 +214,34 @@ public class VarLengthColumnReaders {
int nullsRead;
boolean currentValNull = false;
// store a hard reference to the vector (which is also stored in the superclass) to prevent repetitive casting
- protected NullableVarCharVector nullableVarCharVector;
+ protected final NullableVarCharVector.Mutator mutator;
+ private final NullableVarCharVector vector;
NullableVarCharColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableVarCharVector v,
SchemaElement schemaElement) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
- nullableVarCharVector = v;
+ vector = v;
+ this.mutator = vector.getMutator();
}
- public boolean setSafe(int index, ByteBuf value, int start, int length) {
+ public boolean setSafe(int index, DrillBuf value, int start, int length) {
boolean success;
- if(index >= nullableVarCharVector.getValueCapacity()) return false;
+ if(index >= vector.getValueCapacity()) return false;
if (usingDictionary) {
- ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
- int st=0;
- int len=currDictVal.length();
- NullableVarCharHolder holder = new NullableVarCharHolder();
- holder.buffer=b;
- holder.start=0;
- holder.end=currDictVal.length();
- success = nullableVarCharVector.getMutator().setSafe(index, holder);
- holder.isSet=1;
+ DrillBuf b = DrillBuf.wrapByteBuffer(currDictVal.toByteBuffer());
+ success = mutator.setSafe(index, 1, 0, currDictVal.length(), b);
}
else {
- NullableVarCharHolder holder = new NullableVarCharHolder();
- holder.buffer=value;
- holder.start=start;
- holder.end=start+length;
- holder.isSet=1;
- success = nullableVarCharVector.getMutator().setSafe(index, holder);
+ success = mutator.setSafe(index, 1, start, start+length, value);
}
return success;
}
@Override
public int capacity() {
- return nullableVarCharVector.getData().capacity();
+ return vector.getData().capacity();
}
}
@@ -265,12 +258,12 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, ByteBuf value, int start, int length) {
+ public boolean setSafe(int index, DrillBuf value, int start, int length) {
boolean success;
if(index >= varBinaryVector.getValueCapacity()) return false;
if (usingDictionary) {
- ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
+ DrillBuf b = DrillBuf.wrapByteBuffer(currDictVal.toByteBuffer());
int st=0;
int len=currDictVal.length();
VarBinaryHolder holder = new VarBinaryHolder();
@@ -309,14 +302,12 @@ public class VarLengthColumnReaders {
nullableVarBinaryVector = v;
}
- public boolean setSafe(int index, ByteBuf value, int start, int length) {
+ public boolean setSafe(int index, DrillBuf value, int start, int length) {
boolean success;
if(index >= nullableVarBinaryVector.getValueCapacity()) return false;
if (usingDictionary) {
- ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
- int st=0;
- int len=currDictVal.length();
+ DrillBuf b = DrillBuf.wrapByteBuffer(currDictVal.toByteBuffer());
NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
holder.buffer=b;
holder.start=0;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
index 7f16c64..829b44a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
@@ -18,9 +18,12 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
+
import parquet.bytes.BytesUtils;
import parquet.column.ColumnDescriptor;
import parquet.format.Encoding;
@@ -49,7 +52,7 @@ public abstract class VarLengthValuesColumn<V extends ValueVector> extends VarLe
}
}
- public abstract boolean setSafe(int index, ByteBuf bytes, int start, int length);
+ public abstract boolean setSafe(int index, DrillBuf bytes, int start, int length);
@Override
protected void readField(long recordToRead) {
@@ -80,8 +83,7 @@ public abstract class VarLengthValuesColumn<V extends ValueVector> extends VarLe
protected boolean readAndStoreValueSizeInformation() throws IOException {
// re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
try {
- dataTypeLengthInBits = BytesUtils.readIntLittleEndian(pageReader.pageDataByteArray.nioBuffer(),
- (int) pageReader.readyToReadPosInBytes);
+ dataTypeLengthInBits = pageReader.pageDataByteArray.getInt((int) pageReader.readyToReadPosInBytes);
} catch (Throwable t) {
throw t;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
index f9bac65..c6310b1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
@@ -17,12 +17,11 @@
*/
package org.apache.drill.exec.store.parquet2;
-import com.google.common.collect.Lists;
+import io.netty.buffer.DrillBuf;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
+import java.math.BigDecimal;
+import java.util.List;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.DateHolder;
@@ -37,8 +36,9 @@ import org.apache.drill.exec.expr.holders.TimeHolder;
import org.apache.drill.exec.expr.holders.TimeStampHolder;
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.ParquetOutputRecordWriter;
-import org.apache.drill.exec.store.parquet.columnreaders.NullableFixedByteAlignedReaders;
+import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BigIntWriter;
@@ -68,30 +68,31 @@ import parquet.schema.PrimitiveType;
import parquet.schema.Type;
import parquet.schema.Type.Repetition;
-import java.math.BigDecimal;
-import java.util.List;
+import com.google.common.collect.Lists;
public class DrillParquetGroupConverter extends GroupConverter {
private List<Converter> converters;
private MapWriter mapWriter;
+ private final OutputMutator mutator;
- public DrillParquetGroupConverter(ComplexWriterImpl complexWriter, MessageType schema) {
- this(complexWriter.rootAsMap(), schema);
+ public DrillParquetGroupConverter(OutputMutator mutator, ComplexWriterImpl complexWriter, MessageType schema) {
+ this(mutator, complexWriter.rootAsMap(), schema);
}
- public DrillParquetGroupConverter(MapWriter mapWriter, GroupType schema) {
+ public DrillParquetGroupConverter(OutputMutator mutator, MapWriter mapWriter, GroupType schema) {
this.mapWriter = mapWriter;
+ this.mutator = mutator;
converters = Lists.newArrayList();
for (Type type : schema.getFields()) {
Repetition rep = type.getRepetition();
boolean isPrimitive = type.isPrimitive();
if (!isPrimitive) {
if (rep != Repetition.REPEATED) {
- DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mapWriter.map(type.getName()), type.asGroupType());
+ DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mutator, mapWriter.map(type.getName()), type.asGroupType());
converters.add(converter);
} else {
- DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mapWriter.list(type.getName()).map(), type.asGroupType());
+ DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mutator, mapWriter.list(type.getName()).map(), type.asGroupType());
converters.add(converter);
}
} else {
@@ -162,21 +163,21 @@ public class DrillParquetGroupConverter extends GroupConverter {
case BINARY: {
if (type.getOriginalType() == null) {
VarBinaryWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).varBinary() : mapWriter.varBinary(name);
- return new DrillVarBinaryConverter(writer);
+ return new DrillVarBinaryConverter(writer, mutator.getManagedBuffer());
}
switch(type.getOriginalType()) {
case UTF8: {
VarCharWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).varChar() : mapWriter.varChar(name);
- return new DrillVarCharConverter(writer);
+ return new DrillVarCharConverter(writer, mutator.getManagedBuffer());
}
case DECIMAL: {
DecimalMetadata metadata = type.getDecimalMetadata();
if (metadata.getPrecision() <= 28) {
Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name);
- return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale());
+ return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
} else {
Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name);
- return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale());
+ return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
}
}
default: {
@@ -363,16 +364,18 @@ public class DrillParquetGroupConverter extends GroupConverter {
public static class DrillVarBinaryConverter extends PrimitiveConverter {
private VarBinaryWriter writer;
+ private DrillBuf buf;
private VarBinaryHolder holder = new VarBinaryHolder();
- public DrillVarBinaryConverter(VarBinaryWriter writer) {
+ public DrillVarBinaryConverter(VarBinaryWriter writer, DrillBuf buf) {
this.writer = writer;
+ this.buf = buf;
}
@Override
public void addBinary(Binary value) {
- ByteBuf buf = Unpooled.wrappedBuffer(value.toByteBuffer());
- holder.buffer = buf;
+ holder.buffer = buf = buf.reallocIfNeeded(value.length());
+ buf.setBytes(0, value.toByteBuffer());
holder.start = 0;
holder.end = value.length();
writer.write(holder);
@@ -382,15 +385,17 @@ public class DrillParquetGroupConverter extends GroupConverter {
public static class DrillVarCharConverter extends PrimitiveConverter {
private VarCharWriter writer;
private VarCharHolder holder = new VarCharHolder();
+ private DrillBuf buf;
- public DrillVarCharConverter(VarCharWriter writer) {
+ public DrillVarCharConverter(VarCharWriter writer, DrillBuf buf) {
this.writer = writer;
+ this.buf = buf;
}
@Override
public void addBinary(Binary value) {
- ByteBuf buf = Unpooled.wrappedBuffer(value.toByteBuffer());
- holder.buffer = buf;
+ holder.buffer = buf = buf.reallocIfNeeded(value.length());
+ buf.setBytes(0, value.toByteBuffer());
holder.start = 0;
holder.end = value.length();
writer.write(holder);
@@ -400,9 +405,11 @@ public class DrillParquetGroupConverter extends GroupConverter {
public static class DrillBinaryToDecimal28Converter extends PrimitiveConverter {
private Decimal28SparseWriter writer;
private Decimal28SparseHolder holder = new Decimal28SparseHolder();
+ private DrillBuf buf;
- public DrillBinaryToDecimal28Converter(Decimal28SparseWriter writer, int precision, int scale) {
+ public DrillBinaryToDecimal28Converter(Decimal28SparseWriter writer, int precision, int scale, DrillBuf buf) {
this.writer = writer;
+ this.buf = buf.reallocIfNeeded(28);
holder.precision = precision;
holder.scale = scale;
}
@@ -410,7 +417,6 @@ public class DrillParquetGroupConverter extends GroupConverter {
@Override
public void addBinary(Binary value) {
BigDecimal bigDecimal = DecimalUtility.getBigDecimalFromByteArray(value.getBytes(), 0, value.length(), holder.scale);
- ByteBuf buf = Unpooled.wrappedBuffer(new byte[28]);
DecimalUtility.getSparseFromBigDecimal(bigDecimal, buf, 0, holder.scale, holder.precision, Decimal28SparseHolder.nDecimalDigits);
holder.buffer = buf;
writer.write(holder);
@@ -420,9 +426,11 @@ public class DrillParquetGroupConverter extends GroupConverter {
public static class DrillBinaryToDecimal38Converter extends PrimitiveConverter {
private Decimal38SparseWriter writer;
private Decimal38SparseHolder holder = new Decimal38SparseHolder();
+ private DrillBuf buf;
- public DrillBinaryToDecimal38Converter(Decimal38SparseWriter writer, int precision, int scale) {
+ public DrillBinaryToDecimal38Converter(Decimal38SparseWriter writer, int precision, int scale, DrillBuf buf) {
this.writer = writer;
+ this.buf = buf.reallocIfNeeded(38);
holder.precision = precision;
holder.scale = scale;
}
@@ -430,7 +438,6 @@ public class DrillParquetGroupConverter extends GroupConverter {
@Override
public void addBinary(Binary value) {
BigDecimal bigDecimal = DecimalUtility.getBigDecimalFromByteArray(value.getBytes(), 0, value.length(), holder.scale);
- ByteBuf buf = Unpooled.wrappedBuffer(new byte[38]);
DecimalUtility.getSparseFromBigDecimal(bigDecimal, buf, 0, holder.scale, holder.precision, Decimal38SparseHolder.nDecimalDigits);
holder.buffer = buf;
writer.write(holder);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index f47acab..16f520c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -156,7 +156,7 @@ public class DrillParquetReader implements RecordReader {
}
writer = new VectorContainerWriter(output);
- recordMaterializer = new DrillParquetRecordMaterializer(writer, projection);
+ recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection);
primitiveVectors = writer.getMapVector().getPrimitiveVectors();
recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
index 69893dc..f9c3480 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
@@ -17,24 +17,21 @@
*/
package org.apache.drill.exec.store.parquet2;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.MapVector;
-import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
+import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
import parquet.io.api.GroupConverter;
import parquet.io.api.RecordMaterializer;
import parquet.schema.MessageType;
-import java.util.List;
-
public class DrillParquetRecordMaterializer extends RecordMaterializer<Void> {
public DrillParquetGroupConverter root;
private ComplexWriter complexWriter;
- public DrillParquetRecordMaterializer(ComplexWriter complexWriter, MessageType schema) {
+ public DrillParquetRecordMaterializer(OutputMutator mutator, ComplexWriter complexWriter, MessageType schema) {
this.complexWriter = complexWriter;
- root = new DrillParquetGroupConverter(complexWriter.rootAsMap(), schema);
+ root = new DrillParquetGroupConverter(mutator, complexWriter.rootAsMap(), schema);
}
public void setPosition(int position) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
index 9a5920d..48e09aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
@@ -18,8 +18,10 @@
package org.apache.drill.exec.store.pojo;
import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
import java.sql.Timestamp;
import java.util.Iterator;
+import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -35,8 +37,12 @@ import org.apache.drill.exec.store.pojo.Writers.NBigIntWriter;
import org.apache.drill.exec.store.pojo.Writers.NBooleanWriter;
import org.apache.drill.exec.store.pojo.Writers.NDoubleWriter;
import org.apache.drill.exec.store.pojo.Writers.NIntWriter;
-import org.apache.drill.exec.store.pojo.Writers.StringWriter;
import org.apache.drill.exec.store.pojo.Writers.NTimeStampWriter;
+import org.apache.drill.exec.store.pojo.Writers.StringWriter;
+
+import com.google.common.collect.Lists;
+
+
public class PojoRecordReader<T> implements RecordReader{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PojoRecordReader.class);
@@ -67,42 +73,51 @@ public class PojoRecordReader<T> implements RecordReader{
public void setup(OutputMutator output) throws ExecutionSetupException {
try{
Field[] fields = pojoClass.getDeclaredFields();
- writers = new PojoWriter[fields.length];
- for(int i = 0; i < writers.length; i++){
+ List<PojoWriter> writers = Lists.newArrayList();
+
+ for(int i = 0; i < fields.length; i++){
Field f = fields[i];
- Class<?> type = f.getType();
+ if(Modifier.isStatic(f.getModifiers())) continue;
+
+ Class<?> type = f.getType();
+ PojoWriter w = null;
if(type == int.class){
- writers[i] = new IntWriter(f);
+ w = new IntWriter(f);
}else if(type == Integer.class){
- writers[i] = new NIntWriter(f);
+ w = new NIntWriter(f);
}else if(type == Long.class){
- writers[i] = new NBigIntWriter(f);
+ w = new NBigIntWriter(f);
}else if(type == Boolean.class){
- writers[i] = new NBooleanWriter(f);
+ w = new NBooleanWriter(f);
}else if(type == double.class){
- writers[i] = new DoubleWriter(f);
+ w = new DoubleWriter(f);
}else if(type == Double.class){
- writers[i] = new NDoubleWriter(f);
+ w = new NDoubleWriter(f);
}else if(type.isEnum()){
- writers[i] = new EnumWriter(f);
+ w = new EnumWriter(f, output.getManagedBuffer());
}else if(type == boolean.class){
- writers[i] = new BitWriter(f);
+ w = new BitWriter(f);
}else if(type == long.class){
- writers[i] = new LongWriter(f);
+ w = new LongWriter(f);
}else if(type == String.class){
- writers[i] = new StringWriter(f);
+ w = new StringWriter(f, output.getManagedBuffer());
}else if (type == Timestamp.class) {
- writers[i] = new NTimeStampWriter(f);
+ w = new NTimeStampWriter(f);
}else{
throw new ExecutionSetupException(String.format("PojoRecord reader doesn't yet support conversions from type [%s].", type));
}
- writers[i].init(output);
+ writers.add(w);
+ w.init(output);
}
+
+ this.writers = writers.toArray(new PojoWriter[writers.size()]);
+
}catch(SchemaChangeException e){
throw new ExecutionSetupException("Failure while setting up schema for PojoRecordReader.", e);
}
+
}
private void allocate(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java
index 03732a0..8227695 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java
@@ -17,8 +17,7 @@
*/
package org.apache.drill.exec.store.pojo;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.buffer.DrillBuf;
import java.lang.reflect.Field;
import java.sql.Timestamp;
@@ -34,8 +33,8 @@ import org.apache.drill.exec.vector.NullableBigIntVector;
import org.apache.drill.exec.vector.NullableBitVector;
import org.apache.drill.exec.vector.NullableFloat8Vector;
import org.apache.drill.exec.vector.NullableIntVector;
-import org.apache.drill.exec.vector.NullableVarCharVector;
import org.apache.drill.exec.vector.NullableTimeStampVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
import com.google.common.base.Charsets;
@@ -105,23 +104,20 @@ public class Writers {
}
private abstract static class AbstractStringWriter extends AbstractWriter<NullableVarCharVector>{
- private ByteBuf data;
+ private DrillBuf data;
private final NullableVarCharHolder h = new NullableVarCharHolder();
- public AbstractStringWriter(Field field) {
+ public AbstractStringWriter(Field field, DrillBuf managedBuf) {
super(field, Types.optional(MinorType.VARCHAR));
+ this.data = managedBuf;
ensureLength(100);
}
void ensureLength(int len){
- if(data == null || data.capacity() < len){
- if(data != null) data.release();
- data = UnpooledByteBufAllocator.DEFAULT.buffer(len);
- }
+ data = data.reallocIfNeeded(len);
}
public void cleanup(){
- data.release();
}
public boolean writeString(String s, int outboundIndex) throws IllegalArgumentException, IllegalAccessException {
@@ -145,8 +141,8 @@ public class Writers {
}
public static class EnumWriter extends AbstractStringWriter{
- public EnumWriter(Field field) {
- super(field);
+ public EnumWriter(Field field, DrillBuf managedBuf) {
+ super(field, managedBuf);
if(!field.getType().isEnum()) throw new IllegalStateException();
}
@@ -159,8 +155,8 @@ public class Writers {
}
public static class StringWriter extends AbstractStringWriter {
- public StringWriter(Field field) {
- super(field);
+ public StringWriter(Field field, DrillBuf managedBuf) {
+ super(field, managedBuf);
if(field.getType() != String.class) throw new IllegalStateException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/util/ByteBufUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ByteBufUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ByteBufUtil.java
index e7bdbcf..37e6040 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ByteBufUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ByteBufUtil.java
@@ -17,42 +17,19 @@
*/
package org.apache.drill.exec.util;
-import static java.nio.ByteOrder.LITTLE_ENDIAN;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
+import io.netty.buffer.DrillBuf;
import java.io.DataInput;
import org.apache.drill.common.util.DrillStringUtils;
public class ByteBufUtil {
- /**
- * Creates a wrapped {@link ByteBuf} of {@code size} number of bytes with
- * the Drill's default endianness(LITTLE_ENDIAN).
- *
- * @param size
- * @return
- */
- public static ByteBuf createBuffer(int size) {
- return Unpooled.wrappedBuffer(new byte[size]).order(LITTLE_ENDIAN);
- }
-
- /**
- * Creates a {@link ByteBuf} which wraps the provided byte array with
- * the Drill's default endianness(LITTLE_ENDIAN).
- *
- * @param size
- * @return
- */
- public static ByteBuf createBuffer(byte[] source) {
- return Unpooled.wrappedBuffer(source).order(LITTLE_ENDIAN);
- }
/**
* Verifies that the the space provided in the buffer is of specified size.
* @throws IllegalArgumentException if the specified boundaries do not describe the expected size.
*/
- public static void checkBufferLength(ByteBuf buffer, int start, int end, int requiredLen) {
+ public static void checkBufferLength(DrillBuf buffer, int start, int end, int requiredLen) {
int actualLen = (end - start);
if (actualLen != requiredLen) {
throw new IllegalArgumentException(String.format("Wrong length %d(%d-%d) in the buffer '%s', expected %d.",
@@ -62,7 +39,7 @@ public class ByteBufUtil {
/**
* Modeled after {@code org.apache.hadoop.io.WritableUtils}.
- * We copy the code to avoid wrapping {@link ByteBuf} to/from {@link DataInput}.
+ * We copy the code to avoid wrapping {@link DrillBuf} to/from {@link DataInput}.
*/
public static class HadoopWritables {
/**
@@ -76,10 +53,10 @@ public class ByteBufUtil {
* is negative, with number of bytes that follow are -(v+124). Bytes are
* stored in the high-non-zero-byte-first order.
*
- * @param buffer ByteBuf to read from
+ * @param buffer DrillBuf to read from
* @param i Integer to be serialized
*/
- public static void writeVInt(ByteBuf buffer, int start, int end, int i) {
+ public static void writeVInt(DrillBuf buffer, int start, int end, int i) {
writeVLong(buffer, start, end, i);
}
@@ -94,10 +71,10 @@ public class ByteBufUtil {
* is negative, with number of bytes that follow are -(v+120). Bytes are
* stored in the high-non-zero-byte-first order.
*
- * @param buffer ByteBuf to write to
+ * @param buffer DrillBuf to write to
* @param i Long to be serialized
*/
- public static void writeVLong(ByteBuf buffer, int start, int end, long i) {
+ public static void writeVLong(DrillBuf buffer, int start, int end, long i) {
int availableBytes = (end-start);
if (availableBytes < getVIntSize(i)) {
throw new NumberFormatException("Expected " + getVIntSize(i) + " bytes but the buffer '"
@@ -136,10 +113,10 @@ public class ByteBufUtil {
/**
* Reads a zero-compressed encoded integer from input stream and returns it.
- * @param buffer ByteBuf to read from
+ * @param buffer DrillBuf to read from
* @return deserialized integer from stream.
*/
- public static int readVInt(ByteBuf buffer, int start, int end) {
+ public static int readVInt(DrillBuf buffer, int start, int end) {
long n = readVLong(buffer, start, end);
if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
throw new NumberFormatException("Value " + n + " too long to fit in integer");
@@ -149,10 +126,10 @@ public class ByteBufUtil {
/**
* Reads a zero-compressed encoded long from input stream and returns it.
- * @param buffer ByteBuf to read from
+ * @param buffer DrillBuf to read from
* @return deserialized long from stream.
*/
- public static long readVLong(ByteBuf buffer, int start, int end) {
+ public static long readVLong(DrillBuf buffer, int start, int end) {
buffer.readerIndex(start);
byte firstByte = buffer.readByte();
int len = decodeVIntSize(firstByte);
[02/47] git commit: DRILL-1281: Read into Direct Memory in Parquet
Reader. Requires Hadoop 2.4 or above
Posted by ja...@apache.org.
DRILL-1281: Read into Direct Memory in Parquet Reader. Requires Hadoop 2.4 or above
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/02e4824e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/02e4824e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/02e4824e
Branch: refs/heads/master
Commit: 02e4824ed9da6e15b5f974f16b15b408695c8ada
Parents: 654c879
Author: Parth Chandra <pc...@maprtech.com>
Authored: Mon Jul 28 13:44:52 2014 -0700
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Mon Aug 18 14:46:26 2014 -0700
----------------------------------------------------------------------
.../drill/common/util/DecimalUtility.java | 9 +-
contrib/storage-hbase/pom.xml | 48 +++++
.../exec/store/hbase/HBaseRecordReader.java | 14 ++
.../drill/exec/store/hive/HiveRecordReader.java | 13 ++
exec/java-exec/pom.xml | 112 ++++++++++--
.../codegen/templates/HolderReaderImpl.java | 1 -
.../drill/exec/physical/impl/ScanBatch.java | 6 +-
.../exec/store/LocalSyncableFileSystem.java | 12 ++
.../apache/drill/exec/store/RecordReader.java | 9 +
.../exec/store/easy/json/JSONRecordReader2.java | 14 ++
.../drill/exec/store/mock/MockRecordReader.java | 13 ++
.../exec/store/parquet/ColumnDataReader.java | 26 ++-
.../store/parquet/ParquetScanBatchCreator.java | 8 +-
.../store/parquet/columnreaders/BitReader.java | 11 +-
.../parquet/columnreaders/ColumnReader.java | 8 +-
.../columnreaders/FixedByteAlignedReader.java | 15 +-
.../NullableFixedByteAlignedReaders.java | 33 ++--
.../NullableVarLengthValuesColumn.java | 5 +-
.../store/parquet/columnreaders/PageReader.java | 127 ++++++++++---
.../columnreaders/ParquetRecordReader.java | 13 ++
.../columnreaders/VarLengthColumnReaders.java | 98 +++++++---
.../columnreaders/VarLengthValuesColumn.java | 5 +-
.../exec/store/parquet2/DrillParquetReader.java | 9 +
.../drill/exec/store/pojo/PojoRecordReader.java | 10 ++
.../exec/store/text/DrillTextRecordReader.java | 14 +-
.../parquet/hadoop/CodecFactoryExposer.java | 96 +++++++++-
.../physical/impl/writer/TestParquetWriter.java | 4 +-
exec/jdbc/pom.xml | 14 +-
pom.xml | 180 +++++++++++--------
29 files changed, 747 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
index 8311d82..85ba918 100644
--- a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java
@@ -553,13 +553,20 @@ public class DecimalUtility {
return 0;
}
-
public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
byte[] value = Arrays.copyOfRange(bytes, start, start + length);
BigInteger unscaledValue = new BigInteger(value);
return new BigDecimal(unscaledValue, scale);
}
+
+ public static BigDecimal getBigDecimalFromByteBuf(ByteBuf bytebuf, int start, int length, int scale) {
+ byte[] value = new byte[length];
+ bytebuf.getBytes(start, value, 0, length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
public static void roundDecimal(ByteBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(desiredScale);
int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(currentScale);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/contrib/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml
index c6043a3..82bf4bb 100644
--- a/contrib/storage-hbase/pom.xml
+++ b/contrib/storage-hbase/pom.xml
@@ -37,6 +37,12 @@
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>hadoop-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Test dependencies -->
@@ -46,6 +52,17 @@
<classifier>tests</classifier>
<version>${project.version}</version>
<scope>test</scope>
+ <!-- here we explicitly exclude hadoop v2.x dependencies
+ because HBase 0.94.x unit tests do not work with hadoop 2
+ test classes, e.g. MiniDFSCluster.
+ This will change once we move to HBase 0.98 or later.
+ -->
+ <exclusions>
+ <exclusion>
+ <artifactId>hadoop-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
@@ -146,10 +163,37 @@
</exclusions>
</dependency>
<dependency>
+ <!-- contrary to other Drill modules, this HBase storage module
+ compiles against hadoop 1.x to allow running a mini HBase
+ cluster to run the unit tests.
+ This will change once we move to HBase 0.98 or later.
+ -->
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>1.2.1</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- test dependencies -->
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
<version>1.2.1</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -230,6 +274,10 @@
<artifactId>jruby-complete</artifactId>
<groupId>org.jruby</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>hadoop-core</artifactId>
+ <groupId>org.apache</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
index 204d486..954cc5a 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
@@ -33,6 +33,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
@@ -70,12 +71,16 @@ public class HBaseRecordReader implements RecordReader, DrillHBaseConstants {
private Scan hbaseScan;
private Configuration hbaseConf;
private Result leftOver;
+ private FragmentContext fragmentContext;
+ private OperatorContext operatorContext;
+
public HBaseRecordReader(Configuration conf, HBaseSubScan.HBaseSubScanSpec subScanSpec,
List<SchemaPath> projectedColumns, FragmentContext context) throws OutOfMemoryException {
hbaseConf = conf;
hbaseTable = subScanSpec.getTableName();
hbaseScan = new Scan(subScanSpec.getStartRow(), subScanSpec.getStopRow());
+ fragmentContext=context;
boolean rowKeyOnly = true;
this.columns = Sets.newLinkedHashSet();
if (projectedColumns != null && projectedColumns.size() != 0) {
@@ -117,6 +122,15 @@ public class HBaseRecordReader implements RecordReader, DrillHBaseConstants {
hbaseScan.setCaching(TARGET_RECORD_COUNT);
}
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
+
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
this.outputMutator = output;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index ac0fe36..36e55d9 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -34,6 +34,7 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
@@ -101,6 +102,9 @@ public class HiveRecordReader implements RecordReader {
protected Object redoRecord;
protected boolean empty;
private Map<String, String> hiveConfigOverride;
+ private FragmentContext fragmentContext;
+ private OperatorContext operatorContext;
+
protected static final int TARGET_RECORD_COUNT = 4000;
protected static final int FIELD_SIZE = 50;
@@ -114,6 +118,7 @@ public class HiveRecordReader implements RecordReader {
this.projectedColumns = projectedColumns;
this.empty = (inputSplit == null && partition == null);
this.hiveConfigOverride = hiveConfigOverride;
+ this.fragmentContext=context;
init();
}
@@ -229,6 +234,14 @@ public class HiveRecordReader implements RecordReader {
}
}
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 5465f12..21dfc67 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -49,6 +49,11 @@
<version>2.1</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
<version>2.5.6</version>
@@ -120,31 +125,106 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-column</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.1-drill-r2</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.1-drill-r2</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
</exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-format</artifactId>
+ <version>2.1.1-SNAPSHOT</version>
+ <exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-common</artifactId>
+ <version>1.5.1-drill-r2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-jackson</artifactId>
+ <version>1.5.1-drill-r2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-encoding</artifactId>
+ <version>1.5.1-drill-r2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-generator</artifactId>
+ <version>1.5.1-drill-r2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -267,7 +347,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
@@ -279,7 +359,6 @@
</exclusion>
</exclusions>
</dependency>
-
</dependencies>
</profile>
<profile>
@@ -287,17 +366,28 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.hadoop</groupId>
- <artifactId>maprfs</artifactId>
- <version>1.0.3-mapr-3.0.0</version>
+ <artifactId>maprfs</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20090211</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
- <version>0.94.13-mapr-1401-m7-3.0.2</version>
+ <version>0.94.17-mapr-1405-4.0.0-FCS</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
index 5f718a0..bd862a1 100644
--- a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
+++ b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
@@ -44,7 +44,6 @@ import java.math.BigInteger;
import org.apache.drill.exec.expr.holders.*;
import org.apache.hadoop.io.Text;
import org.joda.time.Period;
-import org.mortbay.jetty.servlet.Holder;
@SuppressWarnings("unused")
public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index eaf26d1..4b7e33e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -95,6 +95,7 @@ public class ScanBatch implements RecordBatch {
throw new ExecutionSetupException("A scan batch must contain at least one reader.");
this.currentReader = readers.next();
this.oContext = new OperatorContext(subScanConfig, context);
+ this.currentReader.setOperatorContext(this.oContext);
this.currentReader.setup(mutator);
this.partitionColumns = partitionColumns.iterator();
this.partitionValues = this.partitionColumns.hasNext() ? this.partitionColumns.next() : null;
@@ -108,11 +109,14 @@ public class ScanBatch implements RecordBatch {
this(subScanConfig, context, readers, Collections.<String[]> emptyList(), Collections.<Integer> emptyList());
}
- @Override
public FragmentContext getContext() {
return context;
}
+ public OperatorContext getOperatorContext() {
+ return oContext;
+ }
+
@Override
public BatchSchema getSchema() {
return schema;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
index 1e55c73..37b9c9d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
@@ -144,6 +144,18 @@ public class LocalSyncableFileSystem extends FileSystem {
}
@Override
+ public void hsync() throws IOException {
+ output.flush();
+ fos.getFD().sync();
+ }
+
+ @Override
+ public void hflush() throws IOException {
+ output.flush();
+ fos.getFD().sync();
+ }
+
+ @Override
public void write(int b) throws IOException {
output.write(b);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
index fb8a014..1745421 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
public interface RecordReader {
@@ -36,6 +37,14 @@ public interface RecordReader {
public abstract void setup(OutputMutator output) throws ExecutionSetupException;
/**
+ * Set the operator context. The Reader can use this to access the operator context and allocate direct memory
+ * if needed
+ * @param operatorContext
+ */
+ public abstract void setOperatorContext(OperatorContext operatorContext);
+
+
+ /**
* Increment record reader forward, writing into the provided output batch.
*
* @return The number of additional records added to the output.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
index 3e2c81c..cd3cfdc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.vector.complex.fn.JsonReader;
@@ -50,11 +51,15 @@ public class JSONRecordReader2 implements RecordReader{
private InputStream stream;
private JsonReaderWithState jsonReader;
private int recordCount;
+ private FragmentContext fragmentContext;
+ private OperatorContext operatorContext;
+
public JSONRecordReader2(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem,
List<SchemaPath> columns) throws OutOfMemoryException {
this.hadoopPath = new Path(inputPath);
this.fileSystem = fileSystem;
+ this.fragmentContext=fragmentContext;
}
@Override
@@ -80,6 +85,15 @@ public class JSONRecordReader2 implements RecordReader{
throw new DrillRuntimeException(sb.toString(), e);
}
+
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
@Override
public int next() {
writer.allocate();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
index c7fc939..81505ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
@@ -47,10 +48,14 @@ public class MockRecordReader implements RecordReader {
private ValueVector[] valueVectors;
private int recordsRead;
private int batchRecordCount;
+ private FragmentContext fragmentContext;
+ private OperatorContext operatorContext;
+
public MockRecordReader(FragmentContext context, MockScanEntry config) throws OutOfMemoryException {
this.context = context;
this.config = config;
+ this.fragmentContext=context;
}
private int getEstimatedRecordSize(MockColumn[] types) {
@@ -69,6 +74,14 @@ public class MockRecordReader implements RecordReader {
}
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
index 80fbd80..2567c89 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
@@ -17,9 +17,16 @@
*/
package org.apache.drill.exec.store.parquet;
+import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -27,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import parquet.bytes.BytesInput;
import parquet.format.PageHeader;
import parquet.format.Util;
+import parquet.hadoop.util.CompatibilityUtil;
public class ColumnDataReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.class);
@@ -43,13 +51,26 @@ public class ColumnDataReader {
public PageHeader readPageHeader() throws IOException{
return Util.readPageHeader(input);
}
-
+
public BytesInput getPageAsBytesInput(int pageLength) throws IOException{
byte[] b = new byte[pageLength];
input.read(b);
return new HadoopBytesInput(b);
}
-
+
+ public ByteBuf getPageAsBytesBuf(ByteBuf byteBuf, int pageLength) throws IOException{
+ ByteBuffer directBuffer=byteBuf.nioBuffer(0, pageLength);
+ int l=directBuffer.remaining();
+ int bl=byteBuf.capacity();
+ try{
+ CompatibilityUtil.getBuf(input, directBuffer, pageLength);
+ }catch(Exception e) {
+ logger.error("Failed to read data into Direct ByteBuffer with exception: "+e.getMessage());
+ throw new DrillRuntimeException(e.getMessage());
+ }
+ return byteBuf;
+ }
+
public void clear(){
try{
input.close();
@@ -87,4 +108,5 @@ public class ColumnDataReader {
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 5edd3c5..a453e66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -143,7 +143,13 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
}
}
- return new ScanBatch(rowGroupScan, context, readers.iterator(), partitionColumns, selectedPartitionColumns);
+ ScanBatch s = new ScanBatch(rowGroupScan, context, readers.iterator(), partitionColumns, selectedPartitionColumns);
+
+ for(RecordReader r : readers){
+ r.setOperatorContext(s.getOperatorContext());
+ }
+
+ return s;
}
private static boolean isComplex(ParquetMetadata footer) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
index 2c6e488..43e270a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet.columnreaders;
+import io.netty.buffer.ByteBuf;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseDataValueVector;
import org.apache.drill.exec.vector.ValueVector;
@@ -28,7 +29,7 @@ final class BitReader extends ColumnReader {
private byte currentByte;
private byte nextByte;
- private byte[] bytes;
+ private ByteBuf bytebuf;
BitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
boolean fixedLength, ValueVector v, SchemaElement schemaElement) throws ExecutionSetupException {
@@ -45,15 +46,15 @@ final class BitReader extends ColumnReader {
readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
readLength = (int) Math.ceil(readLengthInBits / 8.0);
- bytes = pageReader.pageDataByteArray;
+ bytebuf = pageReader.pageDataByteArray;
// standard read, using memory mapping
if (pageReader.bitShift == 0) {
- ((BaseDataValueVector) valueVec).getData().writeBytes(bytes,
+ ((BaseDataValueVector) valueVec).getData().writeBytes(bytebuf,
(int) readStartInBytes, (int) readLength);
} else { // read in individual values, because a bitshift is necessary with where the last page or batch ended
vectorData = ((BaseDataValueVector) valueVec).getData();
- nextByte = bytes[(int) Math.max(0, Math.ceil(pageReader.valuesRead / 8.0) - 1)];
+ nextByte = bytebuf.getByte((int) Math.max(0, Math.ceil(pageReader.valuesRead / 8.0) - 1));
readLengthInBits = recordsReadInThisIteration + pageReader.bitShift;
int i = 0;
@@ -66,7 +67,7 @@ final class BitReader extends ColumnReader {
// if we are not on the last byte
if ((int) Math.ceil(pageReader.valuesRead / 8.0) + i < pageReader.byteLength) {
// grab the next byte from the buffer, shift and mask it, and OR it with the leftover bits
- nextByte = bytes[(int) Math.ceil(pageReader.valuesRead / 8.0) + i];
+ nextByte = bytebuf.getByte((int) Math.ceil(pageReader.valuesRead / 8.0) + i);
currentByte = (byte) (currentByte | nextByte
<< (8 - pageReader.bitShift)
& ParquetRecordReader.endBitMasks[8 - pageReader.bitShift - 1]);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java
index fd672d6..b240407 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReader.java
@@ -100,10 +100,12 @@ public abstract class ColumnReader<V extends ValueVector> {
public void processPages(long recordsToReadInThisPass) throws IOException {
reset();
- do {
- determineSize(recordsToReadInThisPass, 0);
+ if(recordsToReadInThisPass>0) {
+ do {
+ determineSize(recordsToReadInThisPass, 0);
- } while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReader.currentPage != null);
+ } while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReader.currentPage != null);
+ }
valueVec.getMutator().setValueCount(valuesReadInCurrentPass);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
index 4513aaa..5c25e11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet.columnreaders;
+import io.netty.buffer.ByteBuf;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
@@ -35,7 +36,7 @@ import java.math.BigDecimal;
class FixedByteAlignedReader extends ColumnReader {
- protected byte[] bytes;
+ protected ByteBuf bytebuf;
FixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
@@ -54,13 +55,13 @@ class FixedByteAlignedReader extends ColumnReader {
readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
readLength = (int) Math.ceil(readLengthInBits / 8.0);
- bytes = pageReader.pageDataByteArray;
+ bytebuf = pageReader.pageDataByteArray;
// vectorData is assigned by the superclass read loop method
writeData();
}
protected void writeData() {
- vectorData.writeBytes(bytes,
+ vectorData.writeBytes(bytebuf,
(int) readStartInBytes, (int) readLength);
}
@@ -81,7 +82,7 @@ class FixedByteAlignedReader extends ColumnReader {
}
/**
- * Reads from bytes, converts, and writes to buffer
+ * Reads from bytebuf, converts, and writes to buffer
* @param start the index in bytes to start reading from
* @param index the index of the ValueVector
*/
@@ -101,7 +102,7 @@ class FixedByteAlignedReader extends ColumnReader {
@Override
void addNext(int start, int index) {
dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(
- NullableFixedByteAlignedReaders.NullableDateReader.readIntLittleEndian(bytes, start)
+ NullableFixedByteAlignedReaders.NullableDateReader.readIntLittleEndian(bytebuf, start)
- ParquetOutputRecordWriter.JULIAN_DAY_EPOC - 0.5));
}
}
@@ -119,7 +120,7 @@ class FixedByteAlignedReader extends ColumnReader {
@Override
void addNext(int start, int index) {
int width = Decimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal28Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), Decimal28SparseHolder.nDecimalDigits);
}
@@ -138,7 +139,7 @@ class FixedByteAlignedReader extends ColumnReader {
@Override
void addNext(int start, int index) {
int width = Decimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal38Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), Decimal38SparseHolder.nDecimalDigits);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
index c1575de..4d7f312 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet.columnreaders;
+import io.netty.buffer.ByteBuf;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.vector.NullableBigIntVector;
@@ -41,7 +42,7 @@ import java.math.BigDecimal;
public class NullableFixedByteAlignedReaders {
static class NullableFixedByteAlignedReader extends NullableColumnReader {
- protected byte[] bytes;
+ protected ByteBuf bytebuf;
NullableFixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v, SchemaElement schemaElement) throws ExecutionSetupException {
@@ -57,16 +58,16 @@ public class NullableFixedByteAlignedReaders {
this.readStartInBytes = pageReader.readPosInBytes;
this.readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
this.readLength = (int) Math.ceil(readLengthInBits / 8.0);
- this.bytes = pageReader.pageDataByteArray;
+ this.bytebuf = pageReader.pageDataByteArray;
// fill in data.
- vectorData.writeBytes(bytes, (int) readStartInBytes, (int) readLength);
+ vectorData.writeBytes(bytebuf, (int) readStartInBytes, (int) readLength);
}
}
static class NullableDictionaryIntReader extends NullableColumnReader<NullableIntVector> {
- private byte[] bytes;
+ private ByteBuf bytebuf;
NullableDictionaryIntReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableIntVector v,
@@ -87,7 +88,7 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryBigIntReader extends NullableColumnReader<NullableBigIntVector> {
- private byte[] bytes;
+ private ByteBuf bytebuf;
NullableDictionaryBigIntReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableBigIntVector v,
@@ -106,7 +107,7 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryFloat4Reader extends NullableColumnReader<NullableFloat4Vector> {
- private byte[] bytes;
+ private ByteBuf bytebuf;
NullableDictionaryFloat4Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableFloat4Vector v,
@@ -125,7 +126,7 @@ public class NullableFixedByteAlignedReaders {
static class NullableDictionaryFloat8Reader extends NullableColumnReader<NullableFloat8Vector> {
- private byte[] bytes;
+ private ByteBuf bytebuf;
NullableDictionaryFloat8Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableFloat8Vector v,
@@ -160,7 +161,7 @@ public class NullableFixedByteAlignedReaders {
this.readStartInBytes = pageReader.readPosInBytes;
this.readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
this.readLength = (int) Math.ceil(readLengthInBits / 8.0);
- this.bytes = pageReader.pageDataByteArray;
+ this.bytebuf = pageReader.pageDataByteArray;
dataTypeLengthInBytes = (int) Math.ceil(dataTypeLengthInBits / 8.0);
for (int i = 0; i < recordsReadInThisIteration; i++) {
@@ -183,15 +184,15 @@ public class NullableFixedByteAlignedReaders {
@Override
void addNext(int start, int index) {
- dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(readIntLittleEndian(bytes, start) - ParquetOutputRecordWriter.JULIAN_DAY_EPOC - 0.5));
+ dateVector.getMutator().set(index, DateTimeUtils.fromJulianDay(readIntLittleEndian(bytebuf, start) - ParquetOutputRecordWriter.JULIAN_DAY_EPOC - 0.5));
}
// copied out of parquet library, didn't want to deal with the uneeded throws statement they had declared
- public static int readIntLittleEndian(byte[] in, int offset) {
- int ch4 = in[offset] & 0xff;
- int ch3 = in[offset + 1] & 0xff;
- int ch2 = in[offset + 2] & 0xff;
- int ch1 = in[offset + 3] & 0xff;
+ public static int readIntLittleEndian(ByteBuf in, int offset) {
+ int ch4 = in.getByte(offset) & 0xff;
+ int ch3 = in.getByte(offset + 1) & 0xff;
+ int ch2 = in.getByte(offset + 2) & 0xff;
+ int ch1 = in.getByte(offset + 3) & 0xff;
return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
}
@@ -210,7 +211,7 @@ public class NullableFixedByteAlignedReaders {
@Override
void addNext(int start, int index) {
int width = NullableDecimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal28Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), NullableDecimal28SparseHolder.nDecimalDigits);
}
@@ -229,7 +230,7 @@ public class NullableFixedByteAlignedReaders {
@Override
void addNext(int start, int index) {
int width = NullableDecimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, dataTypeLengthInBytes, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, dataTypeLengthInBytes, schemaElement.getScale());
DecimalUtility.getSparseFromBigDecimal(intermediate, decimal38Vector.getData(), index * width, schemaElement.getScale(),
schemaElement.getPrecision(), NullableDecimal38SparseHolder.nDecimalDigits);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
index 2be9a37..83812ea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java
@@ -17,6 +17,7 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet.columnreaders;
+import io.netty.buffer.ByteBuf;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.ValueVector;
import parquet.bytes.BytesUtils;
@@ -37,7 +38,7 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
- public abstract boolean setSafe(int index, byte[] value, int start, int length);
+ public abstract boolean setSafe(int index, ByteBuf value, int start, int length);
public abstract int capacity();
@@ -81,7 +82,7 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten
}
else {
// re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
- dataTypeLengthInBits = BytesUtils.readIntLittleEndian(pageReader.pageDataByteArray,
+ dataTypeLengthInBits = BytesUtils.readIntLittleEndian(pageReader.pageDataByteArray.nioBuffer(),
(int) pageReader.readyToReadPosInBytes);
}
// I think this also needs to happen if it is null for the random access
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
index 1d300bb..1687c3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java
@@ -18,7 +18,12 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.store.parquet.ColumnDataReader;
import org.apache.drill.exec.store.parquet.ParquetFormatPlugin;
@@ -37,6 +42,7 @@ import parquet.format.PageHeader;
import parquet.format.PageType;
import parquet.format.Util;
import parquet.hadoop.metadata.ColumnChunkMetaData;
+import parquet.hadoop.metadata.CompressionCodecName;
import parquet.schema.PrimitiveType;
// class to keep track of the read position of variable length columns
@@ -48,7 +54,7 @@ final class PageReader {
// store references to the pages that have been uncompressed, but not copied to ValueVectors yet
Page currentPage;
// buffer to store bytes of current page
- byte[] pageDataByteArray;
+ ByteBuf pageDataByteArray;
// for variable length data we need to keep track of our current position in the page data
// as the values and lengths are intermixed, making random access to the length data impossible
@@ -81,8 +87,12 @@ final class PageReader {
Dictionary dictionary;
PageHeader pageHeader = null;
- PageReader(ColumnReader parentStatus, FileSystem fs, Path path, ColumnChunkMetaData columnChunkMetaData) throws ExecutionSetupException{
+ List<ByteBuf> allocatedBuffers;
+
+ PageReader(ColumnReader parentStatus, FileSystem fs, Path path, ColumnChunkMetaData columnChunkMetaData)
+ throws ExecutionSetupException{
this.parentColumnReader = parentStatus;
+ allocatedBuffers = new ArrayList<ByteBuf>();
long totalByteLength = columnChunkMetaData.getTotalUncompressedSize();
long start = columnChunkMetaData.getFirstDataPageOffset();
@@ -93,11 +103,25 @@ final class PageReader {
f.seek(columnChunkMetaData.getDictionaryPageOffset());
PageHeader pageHeader = Util.readPageHeader(f);
assert pageHeader.type == PageType.DICTIONARY_PAGE;
- BytesInput bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
- .decompress( //
- dataReader.getPageAsBytesInput(pageHeader.compressed_page_size), //
- pageHeader.getUncompressed_page_size(), //
- parentColumnReader.columnChunkMetaData.getCodec());
+
+ BytesInput bytesIn;
+ ByteBuf uncompressedData=allocateBuffer(pageHeader.getUncompressed_page_size());
+ allocatedBuffers.add(uncompressedData);
+ if(parentColumnReader.columnChunkMetaData.getCodec()==CompressionCodecName.UNCOMPRESSED) {
+ dataReader.getPageAsBytesBuf(uncompressedData, pageHeader.compressed_page_size);
+ bytesIn=parentColumnReader.parentReader.getCodecFactoryExposer().getBytesInput(uncompressedData,
+ pageHeader.getUncompressed_page_size());
+ }else{
+ ByteBuf compressedData=allocateBuffer(pageHeader.compressed_page_size);
+ dataReader.getPageAsBytesBuf(compressedData, pageHeader.compressed_page_size);
+ bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
+ .decompress(parentColumnReader.columnChunkMetaData.getCodec(),
+ compressedData,
+ uncompressedData,
+ pageHeader.compressed_page_size,
+ pageHeader.getUncompressed_page_size());
+ compressedData.release();
+ }
DictionaryPage page = new DictionaryPage(
bytesIn,
pageHeader.uncompressed_page_size,
@@ -107,7 +131,8 @@ final class PageReader {
this.dictionary = page.getEncoding().initDictionary(parentStatus.columnDescriptor, page);
}
} catch (IOException e) {
- throw new ExecutionSetupException("Error opening or reading metatdata for parquet file at location: " + path.getName(), e);
+ throw new ExecutionSetupException("Error opening or reading metadata for parquet file at location: "
+ + path.getName(), e);
}
}
@@ -137,11 +162,26 @@ final class PageReader {
do {
pageHeader = dataReader.readPageHeader();
if (pageHeader.getType() == PageType.DICTIONARY_PAGE) {
- BytesInput bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
- .decompress( //
- dataReader.getPageAsBytesInput(pageHeader.compressed_page_size), //
- pageHeader.getUncompressed_page_size(), //
- parentColumnReader.columnChunkMetaData.getCodec());
+
+ //TODO: Handle buffer allocation exception
+ BytesInput bytesIn;
+ ByteBuf uncompressedData=allocateBuffer(pageHeader.getUncompressed_page_size());
+ allocatedBuffers.add(uncompressedData);
+ if( parentColumnReader.columnChunkMetaData.getCodec()== CompressionCodecName.UNCOMPRESSED) {
+ dataReader.getPageAsBytesBuf(uncompressedData, pageHeader.compressed_page_size);
+ bytesIn=parentColumnReader.parentReader.getCodecFactoryExposer().getBytesInput(uncompressedData,
+ pageHeader.getUncompressed_page_size());
+ }else{
+ ByteBuf compressedData=allocateBuffer(pageHeader.compressed_page_size);
+ dataReader.getPageAsBytesBuf(compressedData, pageHeader.compressed_page_size);
+ bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
+ .decompress(parentColumnReader.columnChunkMetaData.getCodec(),
+ compressedData,
+ uncompressedData,
+ pageHeader.compressed_page_size,
+ pageHeader.getUncompressed_page_size());
+ compressedData.release();
+ }
DictionaryPage page = new DictionaryPage(
bytesIn,
pageHeader.uncompressed_page_size,
@@ -152,11 +192,25 @@ final class PageReader {
}
} while (pageHeader.getType() == PageType.DICTIONARY_PAGE);
- BytesInput bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
- .decompress( //
- dataReader.getPageAsBytesInput(pageHeader.compressed_page_size), //
- pageHeader.getUncompressed_page_size(), //
- parentColumnReader.columnChunkMetaData.getCodec());
+ //TODO: Handle buffer allocation exception
+ BytesInput bytesIn;
+ ByteBuf uncompressedData=allocateBuffer(pageHeader.getUncompressed_page_size());
+ allocatedBuffers.add(uncompressedData);
+ if(parentColumnReader.columnChunkMetaData.getCodec()==CompressionCodecName.UNCOMPRESSED) {
+ dataReader.getPageAsBytesBuf(uncompressedData, pageHeader.compressed_page_size);
+ bytesIn=parentColumnReader.parentReader.getCodecFactoryExposer().getBytesInput(uncompressedData,
+ pageHeader.getUncompressed_page_size());
+ }else{
+ ByteBuf compressedData=allocateBuffer(pageHeader.compressed_page_size);
+ dataReader.getPageAsBytesBuf(compressedData, pageHeader.compressed_page_size);
+ bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
+ .decompress(parentColumnReader.columnChunkMetaData.getCodec(),
+ compressedData,
+ uncompressedData,
+ pageHeader.compressed_page_size,
+ pageHeader.getUncompressed_page_size());
+ compressedData.release();
+ }
currentPage = new Page(
bytesIn,
pageHeader.data_page_header.num_values,
@@ -172,12 +226,12 @@ final class PageReader {
return false;
}
- pageDataByteArray = currentPage.getBytes().toByteArray();
+ pageDataByteArray = Unpooled.wrappedBuffer(currentPage.getBytes().toByteBuffer());
readPosInBytes = 0;
if (parentColumnReader.getColumnDescriptor().getMaxRepetitionLevel() > 0) {
repetitionLevels = currentPage.getRlEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.REPETITION_LEVEL);
- repetitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray, (int) readPosInBytes);
+ repetitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray.nioBuffer(), (int) readPosInBytes);
// we know that the first value will be a 0, at the end of each list of repeated values we will hit another 0 indicating
// a new record, although we don't know the length until we hit it (and this is a one way stream of integers) so we
// read the first zero here to simplify the reading processes, and start reading the first value the same as all
@@ -191,23 +245,23 @@ final class PageReader {
if (!currentPage.getValueEncoding().usesDictionary()) {
parentColumnReader.usingDictionary = false;
definitionLevels = currentPage.getDlEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.DEFINITION_LEVEL);
- definitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray, (int) readPosInBytes);
+ definitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray.nioBuffer(), (int) readPosInBytes);
readPosInBytes = definitionLevels.getNextOffset();
if (parentColumnReader.columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN) {
valueReader = currentPage.getValueEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.VALUES);
- valueReader.initFromPage(currentPage.getValueCount(), pageDataByteArray, (int) readPosInBytes);
+ valueReader.initFromPage(currentPage.getValueCount(), pageDataByteArray.nioBuffer(), (int) readPosInBytes);
}
} else {
parentColumnReader.usingDictionary = true;
definitionLevels = currentPage.getDlEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.DEFINITION_LEVEL);
- definitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray, (int) readPosInBytes);
+ definitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray.nioBuffer(), (int) readPosInBytes);
readPosInBytes = definitionLevels.getNextOffset();
// initialize two of the dictionary readers, one is for determining the lengths of each value, the second is for
// actually copying the values out into the vectors
dictionaryLengthDeterminingReader = new DictionaryValuesReader(dictionary);
- dictionaryLengthDeterminingReader.initFromPage(currentPage.getValueCount(), pageDataByteArray, (int) readPosInBytes);
+ dictionaryLengthDeterminingReader.initFromPage(currentPage.getValueCount(), pageDataByteArray.nioBuffer(), (int) readPosInBytes);
dictionaryValueReader = new DictionaryValuesReader(dictionary);
- dictionaryValueReader.initFromPage(currentPage.getValueCount(), pageDataByteArray, (int) readPosInBytes);
+ dictionaryValueReader.initFromPage(currentPage.getValueCount(), pageDataByteArray.nioBuffer(), (int) readPosInBytes);
this.parentColumnReader.usingDictionary = true;
}
}
@@ -221,5 +275,28 @@ final class PageReader {
public void clear(){
this.dataReader.clear();
+ // Free all memory, including fixed length types. (Data is being copied for all types not just var length types)
+ //if(!this.parentColumnReader.isFixedLength) {
+ for (ByteBuf b : allocatedBuffers) {
+ b.release();
+ }
+ //}
}
+
+ /*
+ Allocate direct memory to read data into
+ */
+ private ByteBuf allocateBuffer(int size) {
+ ByteBuf b;
+ try {
+ b = parentColumnReader.parentReader.getOperatorContext().getAllocator().buffer(size);
+ //b = UnpooledByteBufAllocator.DEFAULT.heapBuffer(size);
+ }catch(Exception e){
+ throw new DrillRuntimeException("Unable to allocate "+size+" bytes of memory in the Parquet Reader."+
+ "[Exception: "+e.getMessage()+"]"
+ );
+ }
+ return b;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
index 2228787..4e9ac81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
@@ -31,6 +31,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
@@ -82,6 +83,8 @@ public class ParquetRecordReader implements RecordReader {
private VarLenBinaryReader varLengthReader;
private ParquetMetadata footer;
private List<SchemaPath> columns;
+ private FragmentContext fragmentContext;
+ private OperatorContext operatorContext;
private final CodecFactoryExposer codecFactoryExposer;
int rowGroupIndex;
@@ -107,6 +110,8 @@ public class ParquetRecordReader implements RecordReader {
this.batchSize = batchSize;
this.footer = footer;
this.columns = columns;
+ this.fragmentContext=fragmentContext;
+
}
public CodecFactoryExposer getCodecFactoryExposer() {
@@ -133,6 +138,14 @@ public class ParquetRecordReader implements RecordReader {
return batchSize;
}
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
/**
* @param type a fixed length type from the parquet library enum
* @return the length in pageDataByteArray of the type
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
index 979e8c3..555cb94 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java
@@ -19,8 +19,14 @@ package org.apache.drill.exec.store.parquet.columnreaders;
import java.math.BigDecimal;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.util.DecimalUtility;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
import org.apache.drill.exec.vector.Decimal28SparseVector;
@@ -50,9 +56,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
int width = Decimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= decimal28Vector.getValueCapacity()) {
return false;
}
@@ -79,9 +85,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
int width = Decimal28SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= nullableDecimal28Vector.getValueCapacity()) {
return false;
}
@@ -109,9 +115,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
int width = Decimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= decimal28Vector.getValueCapacity()) {
return false;
}
@@ -138,9 +144,9 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
int width = Decimal38SparseHolder.WIDTH;
- BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteArray(bytes, start, length, schemaElement.getScale());
+ BigDecimal intermediate = DecimalUtility.getBigDecimalFromByteBuf(bytebuf, start, length, schemaElement.getScale());
if (index >= nullableDecimal38Vector.getValueCapacity()) {
return false;
}
@@ -170,16 +176,26 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ public boolean setSafe(int index, ByteBuf bytebuf, int start, int length) {
boolean success;
if(index >= varCharVector.getValueCapacity()) return false;
if (usingDictionary) {
- success = varCharVector.getMutator().setSafe(index, currDictValToWrite.getBytes(),
- 0, currDictValToWrite.length());
+ ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
+ int st=0;
+ int len=currDictVal.length();
+ VarCharHolder holder = new VarCharHolder();
+ holder.buffer=b;
+ holder.start=0;
+ holder.end=currDictVal.length();
+ success = varCharVector.getMutator().setSafe(index, holder);
}
else {
- success = varCharVector.getMutator().setSafe(index, bytes, start, length);
+ VarCharHolder holder = new VarCharHolder();
+ holder.buffer=bytebuf;
+ holder.start=start;
+ holder.end=start+length;
+ success = varCharVector.getMutator().setSafe(index, holder);
}
return success;
}
@@ -204,16 +220,28 @@ public class VarLengthColumnReaders {
nullableVarCharVector = v;
}
- public boolean setSafe(int index, byte[] value, int start, int length) {
+ public boolean setSafe(int index, ByteBuf value, int start, int length) {
boolean success;
if(index >= nullableVarCharVector.getValueCapacity()) return false;
if (usingDictionary) {
- success = nullableVarCharVector.getMutator().setSafe(index, currDictValToWrite.getBytes(),
- 0, currDictValToWrite.length());
+ ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
+ int st=0;
+ int len=currDictVal.length();
+ NullableVarCharHolder holder = new NullableVarCharHolder();
+ holder.buffer=b;
+ holder.start=0;
+ holder.end=currDictVal.length();
+ success = nullableVarCharVector.getMutator().setSafe(index, holder);
+ holder.isSet=1;
}
else {
- success = nullableVarCharVector.getMutator().setSafe(index, value, start, length);
+ NullableVarCharHolder holder = new NullableVarCharHolder();
+ holder.buffer=value;
+ holder.start=start;
+ holder.end=start+length;
+ holder.isSet=1;
+ success = nullableVarCharVector.getMutator().setSafe(index, holder);
}
return success;
}
@@ -237,16 +265,26 @@ public class VarLengthColumnReaders {
}
@Override
- public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ public boolean setSafe(int index, ByteBuf value, int start, int length) {
boolean success;
if(index >= varBinaryVector.getValueCapacity()) return false;
if (usingDictionary) {
- success = varBinaryVector.getMutator().setSafe(index, currDictValToWrite.getBytes(),
- 0, currDictValToWrite.length());
+ ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
+ int st=0;
+ int len=currDictVal.length();
+ VarBinaryHolder holder = new VarBinaryHolder();
+ holder.buffer=b;
+ holder.start=0;
+ holder.end=currDictVal.length();
+ success = varBinaryVector.getMutator().setSafe(index, holder);
}
else {
- success = varBinaryVector.getMutator().setSafe(index, bytes, start, length);
+ VarBinaryHolder holder = new VarBinaryHolder();
+ holder.buffer=value;
+ holder.start=start;
+ holder.end=start+length;
+ success = varBinaryVector.getMutator().setSafe(index, holder);
}
return success;
}
@@ -271,16 +309,28 @@ public class VarLengthColumnReaders {
nullableVarBinaryVector = v;
}
- public boolean setSafe(int index, byte[] value, int start, int length) {
+ public boolean setSafe(int index, ByteBuf value, int start, int length) {
boolean success;
if(index >= nullableVarBinaryVector.getValueCapacity()) return false;
if (usingDictionary) {
- success = nullableVarBinaryVector.getMutator().setSafe(index, currDictValToWrite.getBytes(),
- 0, currDictValToWrite.length());
+ ByteBuf b = Unpooled.wrappedBuffer(currDictVal.toByteBuffer());
+ int st=0;
+ int len=currDictVal.length();
+ NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
+ holder.buffer=b;
+ holder.start=0;
+ holder.end=currDictVal.length();
+ holder.isSet=1;
+ success = nullableVarBinaryVector.getMutator().setSafe(index, holder);
}
else {
- success = nullableVarBinaryVector.getMutator().setSafe(index, value, start, length);
+ NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
+ holder.buffer=value;
+ holder.start=start;
+ holder.end=start+length;
+ holder.isSet=1;
+ success = nullableVarBinaryVector.getMutator().setSafe(index, holder);
}
return success;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
index 092c186..7f16c64 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java
@@ -17,6 +17,7 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet.columnreaders;
+import io.netty.buffer.ByteBuf;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
@@ -48,7 +49,7 @@ public abstract class VarLengthValuesColumn<V extends ValueVector> extends VarLe
}
}
- public abstract boolean setSafe(int index, byte[] bytes, int start, int length);
+ public abstract boolean setSafe(int index, ByteBuf bytes, int start, int length);
@Override
protected void readField(long recordToRead) {
@@ -79,7 +80,7 @@ public abstract class VarLengthValuesColumn<V extends ValueVector> extends VarLe
protected boolean readAndStoreValueSizeInformation() throws IOException {
// re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
try {
- dataTypeLengthInBits = BytesUtils.readIntLittleEndian(pageReader.pageDataByteArray,
+ dataTypeLengthInBits = BytesUtils.readIntLittleEndian(pageReader.pageDataByteArray.nioBuffer(),
(int) pageReader.readyToReadPosInBytes);
} catch (Throwable t) {
throw t;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index aaeb536..0d2a225 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.parquet.RowGroupReadEntry;
@@ -64,6 +65,8 @@ public class DrillParquetReader implements RecordReader {
private DrillParquetRecordMaterializer recordMaterializer;
private int recordCount;
private List<ValueVector> primitiveVectors;
+ private OperatorContext operatorContext;
+
public DrillParquetReader(ParquetMetadata footer, RowGroupReadEntry entry, List<SchemaPath> columns, Configuration conf) {
this.footer = footer;
@@ -201,4 +204,10 @@ public class DrillParquetReader implements RecordReader {
@Override
public void cleanup() {
}
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
index 1ebd1f5..9a5920d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.pojo.Writers.BitWriter;
@@ -47,12 +48,21 @@ public class PojoRecordReader<T> implements RecordReader{
private PojoWriter[] writers;
private boolean doCurrent;
private T currentPojo;
+ private OperatorContext operatorContext;
public PojoRecordReader(Class<T> pojoClass, Iterator<T> iterator){
this.pojoClass = pojoClass;
this.iterator = iterator;
}
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
try{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
index 28babd1..68921a2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
@@ -31,6 +31,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
@@ -57,7 +58,8 @@ public class DrillTextRecordReader implements RecordReader {
private byte delimiter;
private int targetRecordCount;
private FieldReference ref = new FieldReference(COL_NAME);
- private FragmentContext context;
+ private FragmentContext fragmentContext;
+ private OperatorContext operatorContext;
private RepeatedVarCharVector vector;
private List<Integer> columnIds = Lists.newArrayList();
private LongWritable key;
@@ -67,7 +69,7 @@ public class DrillTextRecordReader implements RecordReader {
private boolean first = true;
public DrillTextRecordReader(FileSplit split, FragmentContext context, char delimiter, List<SchemaPath> columns) {
- this.context = context;
+ this.fragmentContext = context;
this.delimiter = (byte) delimiter;
boolean getEntireRow = false;
@@ -107,6 +109,14 @@ public class DrillTextRecordReader implements RecordReader {
}
}
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ public void setOperatorContext(OperatorContext operatorContext) {
+ this.operatorContext = operatorContext;
+ }
+
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
MaterializedField field = MaterializedField.create(ref, Types.repeated(TypeProtos.MinorType.VARCHAR));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java b/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
index cb36102..7868a53 100644
--- a/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
+++ b/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
@@ -18,9 +18,20 @@
package parquet.hadoop;
import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.HashMap;
+import java.util.Map;
+import io.netty.buffer.ByteBuf;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.DirectDecompressionCodec;
+import org.apache.hadoop.util.ReflectionUtils;
import parquet.bytes.BytesInput;
import parquet.hadoop.CodecFactory.BytesDecompressor;
import parquet.hadoop.metadata.CompressionCodecName;
@@ -28,9 +39,11 @@ import parquet.hadoop.metadata.CompressionCodecName;
public class CodecFactoryExposer{
private CodecFactory codecFactory;
+ private final Map<String, org.apache.hadoop.io.compress.DirectDecompressionCodec> codecByName = new HashMap<String, org.apache.hadoop.io.compress.DirectDecompressionCodec>();
+ private Configuration configuration;
public CodecFactoryExposer(Configuration config){
- codecFactory = new CodecFactory(config);
+ codecFactory = new CodecFactory(config);configuration = config;
}
public CodecFactory getCodecFactory() {
@@ -41,7 +54,84 @@ public class CodecFactoryExposer{
return codecFactory.getDecompressor(codecName).decompress(bytes, uncompressedSize);
}
- public BytesDecompressor getDecompressor(CompressionCodecName codec) {
- return codecFactory.getDecompressor(codec);
+ public BytesInput getBytesInput(ByteBuf uncompressedByteBuf, int uncompressedSize) throws IOException {
+ ByteBuffer outBuffer=uncompressedByteBuf.nioBuffer(0, uncompressedSize);
+ return new HadoopByteBufBytesInput(outBuffer, 0, outBuffer.limit());
+ }
+
+ public BytesInput decompress(CompressionCodecName codecName,
+ ByteBuf compressedByteBuf,
+ ByteBuf uncompressedByteBuf,
+ int compressedSize,
+ int uncompressedSize) throws IOException {
+ ByteBuffer inpBuffer=compressedByteBuf.nioBuffer(0, compressedSize);
+ ByteBuffer outBuffer=uncompressedByteBuf.nioBuffer(0, uncompressedSize);
+ CompressionCodec c = getCodec(codecName);
+
+ Class<?> cx = c.getClass();
+ ClassLoader l = cx.getClassLoader();
+ Class<?>[] inf = cx.getInterfaces();
+
+ DirectDecompressionCodec d = (DirectDecompressionCodec)c;
+
+ if(d!=null) {
+ d.createDirectDecompressor().decompress(inpBuffer, outBuffer);
+ }else{
+ throw new DrillRuntimeException("Cannot create a decompression codec for codec "+codecName.name());
+ }
+ return new HadoopByteBufBytesInput(outBuffer, 0, outBuffer.limit());
+ }
+
+ private DirectDecompressionCodec getCodec(CompressionCodecName codecName) {
+ String codecClassName = codecName.getHadoopCompressionCodecClassName();
+ if (codecClassName == null) {
+ return null;
+ }
+ DirectDecompressionCodec codec = codecByName.get(codecClassName);
+ if (codec != null) {
+ return codec;
+ }
+
+ try {
+ Class<?> codecClass = Class.forName(codecClassName);
+ codec = (DirectDecompressionCodec)ReflectionUtils.newInstance(codecClass, configuration);
+ codecByName.put(codecClassName, codec);
+ return codec;
+ } catch (ClassNotFoundException e) {
+ throw new BadConfigurationException("Class " + codecClassName + " was not found", e);
+ }
+ }
+
+ public class HadoopByteBufBytesInput extends BytesInput{
+
+ private final ByteBuffer byteBuf;
+ private final int length;
+ private final int offset;
+
+ private HadoopByteBufBytesInput(ByteBuffer byteBuf, int offset, int length) {
+ super();
+ this.byteBuf = byteBuf;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public void writeAllTo(OutputStream out) throws IOException {
+ final WritableByteChannel outputChannel = Channels.newChannel(out);
+ byteBuf.position(offset);
+ ByteBuffer tempBuf = byteBuf.slice();
+ tempBuf.limit(length);
+ outputChannel.write(tempBuf);
+ }
+
+ public ByteBuffer toByteBuffer() throws IOException {
+ byteBuf.position(offset);
+ ByteBuffer buf = byteBuf.slice();
+ buf.limit(length);
+ return buf;
+ }
+
+ public long size() {
+ return length;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 89beeb0..d399fa9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -27,6 +27,7 @@ import org.apache.drill.BaseTestQuery;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
@@ -219,7 +220,8 @@ public class TestParquetWriter extends BaseTestQuery {
fs.delete(path, true);
}
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
+// test("ALTER SESSION SET `planner.add_producer_consumer` = false");
String query = String.format("SELECT %s FROM %s", selection, inputTable);
String create = "CREATE TABLE " + outputFile + " AS " + query;
String validateQuery = String.format("SELECT %s FROM " + outputFile, validationSelection);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/02e4824e/exec/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml
index b739db8..b9b3feb 100644
--- a/exec/jdbc/pom.xml
+++ b/exec/jdbc/pom.xml
@@ -127,7 +127,12 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
@@ -138,6 +143,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
+ <version>2.4.1-mapr-4.0.0-SNAPSHOT</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
<scope>test</scope>
</dependency>
</dependencies>
[36/47] git commit: DRILL-1335: Fix merge join operator when compare
null against null value.
Posted by ja...@apache.org.
DRILL-1335: Fix merge join operator when compare null against null value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cd9eaa88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cd9eaa88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cd9eaa88
Branch: refs/heads/master
Commit: cd9eaa88fee4b01706a9237fb160aad5cb59f9c8
Parents: 746a0c7
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Sat Jun 21 16:04:12 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 27 13:33:48 2014 -0700
----------------------------------------------------------------------
.../exec/physical/impl/join/MergeJoinBatch.java | 39 ++---
.../impl/join/TestMergeJoinMulCondition.java | 48 +++++-
.../test/resources/join/merge_join_nullkey.json | 171 +++++++++++++++++++
3 files changed, 228 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cd9eaa88/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 24ca463..7a6273c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -499,40 +499,21 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
cg.getSetupBlock().assign(JExpr._this().ref(incomingRecordBatch), JExpr._this().ref(incomingRightRecordBatch));
ClassGenerator.HoldingContainer compareRightExprHolder = cg.addExpr(materializedRightExpr, false);
- if (compareLeftExprHolder.isOptional() && compareRightExprHolder.isOptional()) {
- // handle null == null
- cg.getEvalBlock()._if(compareLeftExprHolder.getIsSet().eq(JExpr.lit(0))
- .cand(compareRightExprHolder.getIsSet().eq(JExpr.lit(0))))
- ._then()
- ._return(JExpr.lit(0));
-
- // handle null == !null
- cg.getEvalBlock()._if(compareLeftExprHolder.getIsSet().eq(JExpr.lit(0))
- .cor(compareRightExprHolder.getIsSet().eq(JExpr.lit(0))))
- ._then()
- ._return(JExpr.lit(1));
-
- } else if (compareLeftExprHolder.isOptional()) {
- // handle null == required (null is less than any value)
- cg.getEvalBlock()._if(compareLeftExprHolder.getIsSet().eq(JExpr.lit(0)))
- ._then()
- ._return(JExpr.lit(-1));
-
- } else if (compareRightExprHolder.isOptional()) {
- // handle required == null (null is less than any value)
- cg.getEvalBlock()._if(compareRightExprHolder.getIsSet().eq(JExpr.lit(0)))
- ._then()
- ._return(JExpr.lit(1));
- }
-
LogicalExpression fh = FunctionGenerationHelper.getComparator(compareLeftExprHolder,
compareRightExprHolder,
context.getFunctionRegistry());
HoldingContainer out = cg.addExpr(fh, false);
- //If not 0, it means not equal. We return this out value.
- JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
- jc._then()._return(out.getValue());
+ // If not 0, it means not equal. We return this out value.
+ // Null compares to Null should returns null (unknown). In such case, we return 1 to indicate they are not equal.
+ if (compareLeftExprHolder.isOptional() && compareRightExprHolder.isOptional()) {
+ JConditional jc = cg.getEvalBlock()._if(compareLeftExprHolder.getIsSet().eq(JExpr.lit(0)).
+ cand(compareRightExprHolder.getIsSet().eq(JExpr.lit(0))));
+ jc._then()._return(JExpr.lit(1));
+ jc._elseif(out.getValue().ne(JExpr.lit(0)))._then()._return(out.getValue());
+ } else {
+ cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(out.getValue());
+ }
}
//Pass the equality check for all the join conditions. Finally, return 0.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cd9eaa88/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
index d9cfa5c..bf402d1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
@@ -37,7 +37,6 @@ import org.junit.rules.TestRule;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-//@Ignore("Currently returns wrong result. Stopped working when incoming became more than one result set.")
public class TestMergeJoinMulCondition extends PopUnitTestBase {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMergeJoinMulCondition.class);
@@ -72,4 +71,51 @@ public class TestMergeJoinMulCondition extends PopUnitTestBase {
}
}
+ @Test
+ // The physical plan is obtained through sql:
+ // alter session set `planner.enable_hashjoin`=false;
+ // select * from cp.`region.json` t1, cp.`region.json` t2 where t1.non_exist = t2.non_exist2 ;
+ public void testMergeJoinInnerNullKey() throws Exception {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+
+ bit1.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/join/merge_join_nullkey.json"), Charsets.UTF_8).replace("${JOIN_TYPE}", "INNER"));
+ int count = 0;
+ for(QueryResultBatch b : results) {
+ if (b.getHeader().getRowCount() != 0)
+ count += b.getHeader().getRowCount();
+ b.release();
+ }
+ assertEquals(0, count);
+ }
+ }
+
+ @Test
+ // The physical plan is obtained through sql:
+ // alter session set `planner.enable_hashjoin`=false;
+ // select * from cp.`region.json` t1 left outer join cp.`region.json` t2 on t1.non_exist = t2.non_exist2 ;
+ public void testMergeJoinLeftOuterNullKey() throws Exception {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+
+ bit1.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/join/merge_join_nullkey.json"), Charsets.UTF_8).replace("${JOIN_TYPE}", "LEFT"));
+ int count = 0;
+ for(QueryResultBatch b : results) {
+ if (b.getHeader().getRowCount() != 0)
+ count += b.getHeader().getRowCount();
+ b.release();
+ }
+ assertEquals(110, count);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cd9eaa88/exec/java-exec/src/test/resources/join/merge_join_nullkey.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/join/merge_join_nullkey.json b/exec/java-exec/src/test/resources/join/merge_join_nullkey.json
new file mode 100644
index 0000000..b283dda
--- /dev/null
+++ b/exec/java-exec/src/test/resources/join/merge_join_nullkey.json
@@ -0,0 +1,171 @@
+{
+ "head" : {
+ "version" : 1,
+ "generator" : {
+ "type" : "DefaultSqlHandler",
+ "info" : ""
+ },
+ "type" : "APACHE_DRILL_PHYSICAL",
+ "options" : [ ],
+ "queue" : 0,
+ "resultMode" : "EXEC"
+ },
+ "graph" : [ {
+ "pop" : "fs-scan",
+ "@id" : 11,
+ "files" : [ "/region.json" ],
+ "storage" : {
+ "type" : "file",
+ "enabled" : true,
+ "connection" : "classpath:///",
+ "workspaces" : null,
+ "formats" : {
+ "json" : {
+ "type" : "json"
+ },
+ "parquet" : {
+ "type" : "parquet"
+ }
+ }
+ },
+ "format" : {
+ "type" : "json"
+ },
+ "selectionRoot" : "/region.json",
+ "cost" : 18.0
+ }, {
+ "pop" : "project",
+ "@id" : 9,
+ "exprs" : [ {
+ "ref" : "`T2¦¦*`",
+ "expr" : "`*`"
+ } ],
+ "child" : 11,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "external-sort",
+ "@id" : 7,
+ "child" : 9,
+ "orderings" : [ {
+ "order" : "ASC",
+ "expr" : "`T2¦¦non_exist`",
+ "nullDirection" : "UNSPECIFIED"
+ } ],
+ "reverse" : false,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "selection-vector-remover",
+ "@id" : 5,
+ "child" : 7,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "fs-scan",
+ "@id" : 10,
+ "files" : [ "/region.json" ],
+ "storage" : {
+ "type" : "file",
+ "enabled" : true,
+ "connection" : "classpath:///",
+ "workspaces" : null,
+ "formats" : {
+ "json" : {
+ "type" : "json"
+ },
+ "parquet" : {
+ "type" : "parquet"
+ }
+ }
+ },
+ "format" : {
+ "type" : "json"
+ },
+ "selectionRoot" : "/region.json",
+ "cost" : 18.0
+ }, {
+ "pop" : "project",
+ "@id" : 8,
+ "exprs" : [ {
+ "ref" : "`T3¦¦*`",
+ "expr" : "`*`"
+ } ],
+ "child" : 10,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "external-sort",
+ "@id" : 6,
+ "child" : 8,
+ "orderings" : [ {
+ "order" : "ASC",
+ "expr" : "`T3¦¦non_exist2`",
+ "nullDirection" : "UNSPECIFIED"
+ } ],
+ "reverse" : false,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "selection-vector-remover",
+ "@id" : 4,
+ "child" : 6,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "merge-join",
+ "@id" : 3,
+ "left" : 5,
+ "right" : 4,
+ "conditions" : [ {
+ "relationship" : "==",
+ "left" : "`T2¦¦non_exist`",
+ "right" : "`T3¦¦non_exist2`"
+ } ],
+ "joinType" : "${JOIN_TYPE}",
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "project",
+ "@id" : 2,
+ "exprs" : [ {
+ "ref" : "`T2¦¦*`",
+ "expr" : "`T2¦¦*`"
+ }, {
+ "ref" : "`T3¦¦*`",
+ "expr" : "`T3¦¦*`"
+ } ],
+ "child" : 3,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "project",
+ "@id" : 1,
+ "exprs" : [ {
+ "ref" : "`*`",
+ "expr" : "`T2¦¦*`"
+ }, {
+ "ref" : "`*0`",
+ "expr" : "`T3¦¦*`"
+ } ],
+ "child" : 2,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ }, {
+ "pop" : "screen",
+ "@id" : 0,
+ "child" : 1,
+ "initialAllocation" : 1000000,
+ "maxAllocation" : 10000000000,
+ "cost" : 18.0
+ } ]
+}
[43/47] DRILL-1283: JSON project pushdown.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/929d765a/exec/java-exec/src/test/resources/store/json/single_column_long_file.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/single_column_long_file.json b/exec/java-exec/src/test/resources/store/json/single_column_long_file.json
new file mode 100644
index 0000000..e24dbbb
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/single_column_long_file.json
@@ -0,0 +1,13512 @@
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"N."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"F."}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"I."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"K."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"N."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":"P."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"K."}
+{"mi":"M."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"V."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"Z"}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"V."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"D."}
+{"mi":"E."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"O."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"F."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"S"}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"F."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"I."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"O."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"I."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"N."}
+{"mi":"A."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L"}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"I."}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"J"}
+{"mi":null}
+{"mi":"F."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":" "}
+{"mi":"L."}
+{"mi":null}
+{"mi":"Z."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"Dog"}
+{"mi":"J."}
+{"mi":"P."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"H."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"F."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"P."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"M"}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"N."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"F."}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"I."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"K."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"N."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":"P."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"K."}
+{"mi":"M."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"V."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"Z"}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"V."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"D."}
+{"mi":"E."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"O."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"F."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"S"}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"F."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"I."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"O."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"I."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"N."}
+{"mi":"A."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L"}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"I."}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"J"}
+{"mi":null}
+{"mi":"F."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":" "}
+{"mi":"L."}
+{"mi":null}
+{"mi":"Z."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"Dog"}
+{"mi":"J."}
+{"mi":"P."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"H."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"F."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"P."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"M"}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"N."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"F."}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"I."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"K."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"N."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":"P."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"K."}
+{"mi":"M."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"V."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"Z"}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"V."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"D."}
+{"mi":"E."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"O."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"F."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"S"}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"F."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"I."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"O."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"I."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"N."}
+{"mi":"A."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L"}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"I."}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"J"}
+{"mi":null}
+{"mi":"F."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":" "}
+{"mi":"L."}
+{"mi":null}
+{"mi":"Z."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"Dog"}
+{"mi":"J."}
+{"mi":"P."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"H."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"F."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"P."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"M"}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"N."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"F."}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"I."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"K."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"N."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":"P."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"K."}
+{"mi":"M."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"V."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"Z"}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"V."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"D."}
+{"mi":"E."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"O."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"F."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"I."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"O."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"I."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"N."}
+{"mi":"A."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L"}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"I."}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"J"}
+{"mi":null}
+{"mi":"F."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":" "}
+{"mi":"L."}
+{"mi":null}
+{"mi":"Z."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"Dog"}
+{"mi":"J."}
+{"mi":"P."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"H."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"F."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"P."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"M"}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"N."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"F."}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"I."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"K."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"N."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":"P."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"K."}
+{"mi":"M."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"V."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"Z"}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"V."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"D."}
+{"mi":"E."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"O."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"F."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"S"}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"F."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"I."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"O."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"I."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"N."}
+{"mi":"A."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L"}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"I."}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"J"}
+{"mi":null}
+{"mi":"F."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":" "}
+{"mi":"L."}
+{"mi":null}
+{"mi":"Z."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"Dog"}
+{"mi":"J."}
+{"mi":"P."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"H."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"F."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"P."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"M"}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"N."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"F."}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"I."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"K."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"W."}
+{"mi":"N."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":"P."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"K."}
+{"mi":"M."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"V."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"G."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"Z"}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"V."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"H."}
+{"mi":"J."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"D."}
+{"mi":"E."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"T."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"G."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":"G."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"O."}
+{"mi":"C."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"F."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"S"}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"F."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":"I."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"N."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"R."}
+{"mi":"D."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"O."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"C."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"F."}
+{"mi":"W."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"S."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"I."}
+{"mi":"C."}
+{"mi":"V."}
+{"mi":"N."}
+{"mi":"A."}
+{"mi":"S."}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"P."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L"}
+{"mi":"R."}
+{"mi":"F."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"R."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"D."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"B."}
+{"mi":"H."}
+{"mi":"I."}
+{"mi":"E."}
+{"mi":"D."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"E."}
+{"mi":"J."}
+{"mi":"G."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"W."}
+{"mi":"R."}
+{"mi":null}
+{"mi":"D."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"J"}
+{"mi":null}
+{"mi":"F."}
+{"mi":"M."}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"V."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"S."}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"I."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":" "}
+{"mi":"L."}
+{"mi":null}
+{"mi":"Z."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"O."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"T."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"Dog"}
+{"mi":"J."}
+{"mi":"P."}
+{"mi":"B."}
+{"mi":"E."}
+{"mi":"G."}
+{"mi":"T."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"E."}
+{"mi":"S."}
+{"mi":"R."}
+{"mi":"S."}
+{"mi":"C."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"P."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"W."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"H."}
+{"mi":"M."}
+{"mi":"T."}
+{"mi":null}
+{"mi":null}
+{"mi":"E."}
+{"mi":"W."}
+{"mi":"W."}
+{"mi":"C."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"M."}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"C."}
+{"mi":"R."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":"K."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"L."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":"A."}
+{"mi":"L."}
+{"mi":"C."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"Y."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"J."}
+{"mi":"H."}
+{"mi":"F."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"T."}
+{"mi":"P."}
+{"mi":"J."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"E."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"A."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"V."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"L."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"M"}
+{"mi":null}
+{"mi":"L."}
+{"mi":"S."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"R."}
+{"mi":"M."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"W."}
+{"mi":null}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"B."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"H."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"C."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"P."}
+{"mi":"C."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":"D."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"S."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"C."}
+{"mi":"E."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"I."}
+{"mi":null}
+{"mi":"J."}
+{"mi":null}
+{"mi":"F."}
+{"mi":null}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":"M."}
+{"mi":"V."}
+{"mi":"C."}
+{"mi":"A."}
+{"mi":"K."}
+{"mi":"G."}
+{"mi":"M."}
+{"mi":"S."}
+{"mi":"P."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":null}
+{"mi":"N."}
+{"mi":null}
+{"mi":"B."}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"B."}
+{"mi":"J."}
+{"mi":"K."}
+{"mi":null}
+{"mi":null}
+{"mi":"K."}
+{"mi":"K."}
+{"mi":"J."}
+{"mi":"A."}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":"D."}
+{"mi":"B."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"R."}
+{"mi":"L."}
+{"mi":"H."}
+{"mi":"C."}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":null}
+{"mi":"M."}
+{"mi":null}
+{"mi":"A."}
+{"mi":null}
+{"mi":"M."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":null}
+{"mi":"J."}
+{"mi":"L."}
+{"mi":"A."}
+{"mi":"F."}
+{"mi":"A."}
+{"mi":"M."}
+{"mi":"E."}
+{"mi":"A."}
+{"mi":"A."}
+{"mi":"C."}
+{"mi":"L."}
+{"mi":"W."}
+{"mi":"J."}
+{"mi":null}
+{"mi":"A. C."}
+{"mi":"J."}
+{"mi":"W."}
+{"mi":"S."}
+{"mi":null}
+{"mi":"R."}
+{"mi":"W."}
+{"mi":"L."}
+{"mi":null}
+{"mi":"K."}
+{
<TRUNCATED>