You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/28 02:30:40 UTC
[5/5] git commit: DRILL-185: Extend code/clean code generator and add
first aggregate functions.
DRILL-185: Extend code/clean code generator and add first aggregate functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e43093d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e43093d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e43093d9
Branch: refs/heads/master
Commit: e43093d9e0df6a2c3f9f217cf8b16969f13afabb
Parents: daa42b8
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Aug 24 21:21:51 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Aug 27 16:38:28 2013 -0700
----------------------------------------------------------------------
.../common/expression/OutputTypeDeterminer.java | 1 +
.../expression/fn/AggregationFunctions.java | 2 +-
.../templates/FixedValueVectors.java | 29 +-
.../templates/NullableValueVectors.java | 37 ++-
.../templates/RepeatedValueVectors.java | 18 +-
.../templates/VariableLengthVectors.java | 54 +++-
.../drill/exec/compile/ClassTransformer.java | 27 +-
.../exec/compile/TemplateClassDefinition.java | 34 +-
.../compile/sig/DefaultGeneratorSignature.java | 22 --
.../exec/compile/sig/GeneratorMapping.java | 41 ++-
.../drill/exec/compile/sig/MappingSet.java | 37 ++-
.../drill/exec/compile/sig/SignatureHolder.java | 24 +-
.../apache/drill/exec/expr/CodeGenerator.java | 130 ++++++--
.../drill/exec/expr/DirectExpression.java | 50 +++
.../apache/drill/exec/expr/DrillAggFunc.java | 10 +
.../apache/drill/exec/expr/DrillAggrFunc.java | 9 -
.../apache/drill/exec/expr/DrillBatchFunc.java | 10 -
.../org/apache/drill/exec/expr/DrillFunc.java | 5 -
.../apache/drill/exec/expr/DrillSimpleFunc.java | 8 +
.../drill/exec/expr/EvaluationVisitor.java | 68 ++--
.../exec/expr/ExpressionTreeMaterializer.java | 3 +-
.../exec/expr/ValueVectorReadExpression.java | 15 +-
.../exec/expr/ValueVectorWriteExpression.java | 13 +-
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 63 +++-
.../drill/exec/expr/fn/DrillFuncHolder.java | 207 +++++++++----
.../exec/expr/fn/DrillSimpleFuncHolder.java | 103 +++++++
.../drill/exec/expr/fn/FunctionConverter.java | 23 +-
.../drill/exec/expr/fn/FunctionHolder.java | 153 ---------
.../expr/fn/FunctionImplementationRegistry.java | 19 +-
.../exec/expr/fn/agg/impl/CountFunctions.java | 69 +++++
.../exec/expr/fn/agg/impl/SumFunctions.java | 69 +++++
.../drill/exec/expr/fn/impl/Alternator.java | 6 +-
.../drill/exec/expr/fn/impl/ByteSubstring.java | 4 +-
.../drill/exec/expr/fn/impl/CharSubstring.java | 4 +-
.../exec/expr/fn/impl/ComparatorFunctions.java | 6 +-
.../exec/expr/fn/impl/ComparisonFunctions.java | 50 +--
.../fn/impl/ComparisonFunctionsNullable.java | 26 +-
.../apache/drill/exec/expr/fn/impl/Hash.java | 4 +-
.../drill/exec/expr/fn/impl/IsNotNull.java | 11 +-
.../apache/drill/exec/expr/fn/impl/IsNull.java | 11 +-
.../drill/exec/expr/fn/impl/MathFunctions.java | 8 +-
.../expr/fn/impl/SimpleRepeatedFunctions.java | 8 +-
.../drill/exec/expr/holders/Float8Holder.java | 13 -
.../exec/expr/holders/NullableFloat8Holder.java | 14 -
.../drill/exec/physical/base/AbstractBase.java | 3 +
.../physical/base/AbstractPhysicalVisitor.java | 6 +
.../exec/physical/base/PhysicalVisitor.java | 2 +-
.../drill/exec/physical/config/Project.java | 1 -
.../physical/config/StreamingAggregate.java | 83 +++++
.../drill/exec/physical/impl/ImplCreator.java | 12 +-
.../drill/exec/physical/impl/ScanBatch.java | 32 +-
.../exec/physical/impl/aggregate/AggBatch.java | 309 +++++++++++++++++++
.../impl/aggregate/AggBatchCreator.java | 23 ++
.../physical/impl/aggregate/AggTemplate.java | 288 +++++++++++++++++
.../physical/impl/aggregate/Aggregator.java | 26 ++
.../physical/impl/aggregate/BatchIterator.java | 9 +
.../physical/impl/aggregate/InternalBatch.java | 65 ++++
.../physical/impl/filter/FilterEvaluator.java | 10 -
.../physical/impl/filter/FilterTemplate.java | 7 +-
.../exec/physical/impl/filter/Filterer.java | 3 +-
.../partitionsender/OutgoingRecordBatch.java | 71 ++---
.../impl/partitionsender/Partitioner.java | 6 +-
.../partitionsender/PartitionerEvaluator.java | 28 --
.../partitionsender/PartitionerTemplate.java | 7 +-
.../physical/impl/project/ProjectEvaluator.java | 12 -
.../impl/project/ProjectRecordBatch.java | 41 +--
.../exec/physical/impl/project/Projector.java | 6 +-
.../impl/project/ProjectorTemplate.java | 7 +-
.../exec/physical/impl/sort/Comparator.java | 11 -
.../physical/impl/sort/ReadIndexRewriter.java | 87 ------
.../physical/impl/sort/RecordBatchData.java | 2 +-
.../exec/physical/impl/sort/SortBatch.java | 15 +-
.../exec/physical/impl/sort/SortSignature.java | 20 --
.../exec/physical/impl/sort/SortTemplate.java | 7 +-
.../drill/exec/physical/impl/sort/Sorter.java | 3 +-
.../exec/physical/impl/svremover/Copier.java | 10 +-
.../impl/svremover/CopierTemplate2.java | 9 +-
.../impl/svremover/CopierTemplate4.java | 14 +-
.../physical/impl/svremover/CopyEvaluator.java | 11 -
.../impl/svremover/RemovingRecordBatch.java | 66 +---
.../drill/exec/record/HyperVectorWrapper.java | 15 +
.../drill/exec/record/MaterializedField.java | 10 +-
.../drill/exec/record/SimpleVectorWrapper.java | 8 +
.../apache/drill/exec/record/TransferPair.java | 1 +
.../drill/exec/record/VectorContainer.java | 36 ++-
.../apache/drill/exec/record/VectorWrapper.java | 1 +
.../exec/record/selection/SelectionVector4.java | 30 +-
.../drill/exec/store/mock/MockRecordReader.java | 7 +-
.../drill/exec/vector/BaseValueVector.java | 5 +
.../org/apache/drill/exec/vector/BitVector.java | 28 +-
.../apache/drill/exec/vector/ValueVector.java | 4 +
.../vector/allocator/FixedVectorAllocator.java | 24 ++
.../allocator/VariableEstimatedVector.java | 19 ++
.../allocator/VariableVectorAllocator.java | 19 ++
.../exec/vector/allocator/VectorAllocator.java | 30 ++
.../parquet/hadoop/CodecFactoryExposer.java | 6 +-
.../exec/compile/ExampleInternalInterface.java | 24 --
.../exec/compile/TestClassTransformation.java | 5 +-
.../exec/physical/impl/SimpleRootExec.java | 5 +
.../drill/exec/physical/impl/agg/TestAgg.java | 115 +++++++
.../exec/test/generated/TestGeneratedClass.java | 15 -
.../java-exec/src/test/resources/agg/test1.json | 46 +++
.../src/test/resources/agg/twokey.json | 59 ++++
103 files changed, 2337 insertions(+), 1005 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
index 01b49d2..229de3d 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
@@ -28,6 +28,7 @@ public interface OutputTypeDeterminer {
public static OutputTypeDeterminer FIXED_BIT = new FixedType(Types.required(MinorType.BIT));
public static OutputTypeDeterminer FIXED_INT = new FixedType(Types.required(MinorType.INT));
+ public static OutputTypeDeterminer FIXED_BIGINT = new FixedType(Types.required(MinorType.BIGINT));
public MajorType getOutputType(List<LogicalExpression> expressions);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
index 5102624..eed9f1e 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
@@ -28,7 +28,7 @@ public class AggregationFunctions implements CallProvider {
@Override
public FunctionDefinition[] getFunctionDefintions() {
return new FunctionDefinition[] {
- FunctionDefinition.aggregator("count", new AnyTypeAllowed(1), new OutputTypeDeterminer.SameAsFirstInput()),
+ FunctionDefinition.aggregator("count", new AnyTypeAllowed(1), OutputTypeDeterminer.FIXED_BIGINT),
FunctionDefinition.aggregator("sum", new AnyTypeAllowed(1), new OutputTypeDeterminer.SameAsFirstInput())
};
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
index 64e9fc8..2eeb9ed 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
@@ -14,7 +14,10 @@ import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.BitVector;
+import org.apache.drill.common.expression.FieldReference;
+import java.io.ObjectInputStream.GetField;
import java.util.Arrays;
import java.util.Random;
@@ -91,7 +94,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
public TransferPair getTransferPair(){
- return new TransferImpl();
+ return new TransferImpl(getField());
+ }
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().clone(ref));
}
public void transferTo(${minor.class}Vector target){
@@ -104,8 +110,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
private class TransferImpl implements TransferPair{
${minor.class}Vector to;
- public TransferImpl(){
- this.to = new ${minor.class}Vector(getField(), allocator);
+ public TransferImpl(MaterializedField field){
+ this.to = new ${minor.class}Vector(field, allocator);
}
public ${minor.class}Vector getTo(){
@@ -115,6 +121,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
public void transfer(){
transferTo(to);
}
+
+ @Override
+ public void copyValue(int fromIndex, int toIndex) {
+ to.copyFrom(fromIndex, toIndex, ${minor.class}Vector.this);
+ }
}
public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
@@ -214,6 +225,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
}
+ public boolean setSafe(int index, ${minor.class}Holder holder){
+ if(index >= getValueCapacity()) return false;
+ set(index, holder);
+ return true;
+ }
+
void set(int index, Nullable${minor.class}Holder holder){
data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
}
@@ -235,6 +252,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
}
+ public boolean setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ if(index >= getValueCapacity()) return false;
+ set(index, value);
+ return true;
+ }
+
public void set(int index, ${minor.class}Holder holder){
data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
index 6713ae7..a543131 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
@@ -1,5 +1,3 @@
-import java.lang.UnsupportedOperationException;
-
<@pp.dropOutputFile />
<#list types as type>
<#list type.minor as minor>
@@ -11,6 +9,8 @@ import java.lang.UnsupportedOperationException;
package org.apache.drill.exec.vector;
+
+
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import io.netty.buffer.ByteBuf;
@@ -29,6 +29,8 @@ import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.UInt2Vector;
import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.common.expression.FieldReference;
+
import com.google.common.collect.Lists;
import com.google.common.collect.ObjectArrays;
@@ -160,8 +162,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
</#if>
public TransferPair getTransferPair(){
- return new TransferImpl();
+ return new TransferImpl(getField());
}
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().clone(ref));
+ }
+
public void transferTo(Nullable${minor.class}Vector target){
bits.transferTo(target.bits);
@@ -173,8 +179,8 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
private class TransferImpl implements TransferPair{
Nullable${minor.class}Vector to;
- public TransferImpl(){
- this.to = new Nullable${minor.class}Vector(getField(), allocator);
+ public TransferImpl(MaterializedField field){
+ this.to = new Nullable${minor.class}Vector(field, allocator);
}
public Nullable${minor.class}Vector getTo(){
@@ -184,6 +190,11 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void transfer(){
transferTo(to);
}
+
+ @Override
+ public void copyValue(int fromIndex, int toIndex) {
+ to.copyFrom(fromIndex, toIndex, Nullable${minor.class}Vector.this);
+ }
}
public Accessor getAccessor(){
@@ -271,8 +282,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
bits.getMutator().set(index, 1);
values.getMutator().set(index, value);
}
-
-
+
public void setSkipNull(int index, ${minor.class}Holder holder){
values.getMutator().set(index, holder);
}
@@ -291,6 +301,19 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
values.getMutator().set(index, holder);
}
+ public boolean setSafe(int index, <#if type.major == "VarLen">Nullable${minor.class}Holder <#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value){
+ boolean b1 = bits.getMutator().setSafe(index, 1);
+ boolean b2 = values.getMutator().setSafe(index, value);
+ if(b1 && b2){
+ setCount++;
+ return true;
+ }else{
+ return false;
+ }
+
+ }
+
+
public void setValueCount(int valueCount) {
assert valueCount >= 0;
Nullable${minor.class}Vector.this.valueCount = valueCount;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
index a584411..907a3b1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
@@ -1,5 +1,3 @@
-import org.apache.drill.exec.vector.ValueVector;
-
<@pp.dropOutputFile />
<#list types as type>
<#list type.minor as minor>
@@ -24,6 +22,8 @@ import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.common.expression.FieldReference;
+
import java.util.List;
@@ -68,7 +68,10 @@ import com.google.common.collect.ObjectArrays;
}
public TransferPair getTransferPair(){
- return new TransferImpl();
+ return new TransferImpl(getField());
+ }
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().clone(ref));
}
public void transferTo(Repeated${minor.class}Vector target){
@@ -82,8 +85,8 @@ import com.google.common.collect.ObjectArrays;
private class TransferImpl implements TransferPair{
Repeated${minor.class}Vector to;
- public TransferImpl(){
- this.to = new Repeated${minor.class}Vector(getField(), allocator);
+ public TransferImpl(MaterializedField field){
+ this.to = new Repeated${minor.class}Vector(field, allocator);
}
public Repeated${minor.class}Vector getTo(){
@@ -93,6 +96,11 @@ import com.google.common.collect.ObjectArrays;
public void transfer(){
transferTo(to);
}
+
+ @Override
+ public void copyValue(int fromIndex, int toIndex) {
+ to.copyFrom(fromIndex, toIndex, Repeated${minor.class}Vector.this);
+ }
}
public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
index fe69702..a6a6aa4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
@@ -20,6 +20,8 @@ import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.common.expression.FieldReference;
+
import com.google.common.base.Charsets;
import com.google.common.collect.ObjectArrays;
@@ -113,7 +115,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
}
public TransferPair getTransferPair(){
- return new TransferImpl();
+ return new TransferImpl(getField());
+ }
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().clone(ref));
}
public void transferTo(${minor.class}Vector target){
@@ -145,7 +150,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
if(data.capacity() < outputStart + len) return false;
- from.data.getBytes(start, from.data, outputStart, len);
+ from.data.getBytes(start, data, outputStart, len);
offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, len);
return true;
@@ -155,8 +160,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
private class TransferImpl implements TransferPair{
${minor.class}Vector to;
- public TransferImpl(){
- this.to = new ${minor.class}Vector(getField(), allocator);
+ public TransferImpl(MaterializedField field){
+ this.to = new ${minor.class}Vector(field, allocator);
}
public ${minor.class}Vector getTo(){
@@ -166,6 +171,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public void transfer(){
transferTo(to);
}
+
+ @Override
+ public void copyValue(int fromIndex, int toIndex) {
+ to.copyFrom(fromIndex, toIndex, ${minor.class}Vector.this);
+ }
}
public void allocateNew(int totalBytes, int valueCount) {
@@ -246,6 +256,42 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
data.setBytes(currentOffset, bytes);
}
+
+ public boolean setSafe(int index, Nullable${minor.class}Holder holder){
+ assert holder.isSet == 1;
+ if(index >= getValueCapacity()) return false;
+
+ int start = holder.start;
+ int end = holder.end;
+ int len = end - start;
+
+ int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+ if(data.capacity() < outputStart + len) return false;
+
+ holder.buffer.getBytes(start, data, outputStart, len);
+ offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (index+1) * ${type.width}, len);
+
+ return true;
+ }
+
+ public boolean setSafe(int index, ${minor.class}Holder holder){
+ if(index >= getValueCapacity()) return false;
+
+ int start = holder.start;
+ int end = holder.end;
+ int len = end - start;
+
+ int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+ if(data.capacity() < outputStart + len) return false;
+
+ holder.buffer.getBytes(start, data, outputStart, len);
+ offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (index+1) * ${type.width}, len);
+
+ return true;
+ }
+
public void set(int index, int start, int length, ByteBuf buffer){
assert index >= 0;
int currentOffset = offsetVector.getAccessor().get(index);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
index d7cde2a..d549db4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.util.Iterator;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -45,11 +44,11 @@ import org.objectweb.asm.tree.FieldNode;
import org.objectweb.asm.tree.MethodNode;
import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Sets;
-import com.google.common.io.Files;
import com.google.common.io.Resources;
public class ClassTransformer {
@@ -94,7 +93,6 @@ public class ClassTransformer {
try {
String classBody = ClassBodyBuilder.newBuilder() //
.setClassName(materializedClassName) //
- .setImplementedInterfaces(templateDefinition.getInternalInterface()) //
.setBody(internalClassBody) //
.build();
return getImplementationClass(classLoader, templateDefinition, classBody, materializedClassName);
@@ -112,13 +110,14 @@ public class ClassTransformer {
String materializedClassName) throws ClassTransformationException {
try {
- long t0 = System.nanoTime();
+ Stopwatch t1 = new Stopwatch().start();
final byte[] implementationClass = classLoader.getClassByteCode(materializedClassName, entireClass);
// Get Template Class
final String templateClassName = templateDefinition.getTemplateClassName().replaceAll("\\.", File.separator);
final String templateClassPath = File.separator + templateClassName + ".class";
- long t1 = System.nanoTime();
+ t1.stop();
+ Stopwatch t2 = new Stopwatch().start();
final byte[] templateClass = getClassByteCodeFromPath(templateClassPath);
// int fileNum = new Random().nextInt(100);
//Files.write(templateClass, new File(String.format("/tmp/%d-template.class", fileNum)));
@@ -130,11 +129,13 @@ public class ClassTransformer {
String materializedSlashName = materializedClassName.replace('.', '/');
RemapClasses remapper = new RemapClasses(oldTemplateSlashName, materializedSlashName);
+ Stopwatch t3;
{
ClassNode impl = getClassNodeFromByteCode(implementationClass);
- long t2 = System.nanoTime();
- logger.debug("Compile {}, decode template {}", (t1 - t0)/1000/1000, (t2- t1)/1000/1000);
+ t2.stop();
+ t3 = new Stopwatch().start();
+
ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
@@ -150,7 +151,8 @@ public class ClassTransformer {
// Load the class
classLoader.injectByteCode(materializedClassName, outputClass);
}
-
+ t3.stop();
+ Stopwatch t4 = new Stopwatch().start();
int i = 0;
for (String s : remapper.getSubclasses()) {
logger.debug("Setting up sub class {}", s);
@@ -168,14 +170,15 @@ public class ClassTransformer {
// Files.write(subcw.toByteArray(), new File(String.format("/tmp/%d-sub-%d.class", fileNum, i)));
i++;
}
-
+ t4.stop();
+ logger.debug(String.format("[Compile Time] Janino: %dms, Bytecode load and parse: %dms, Class Merge: %dms, Subclass remap and load: %dms.", t1.elapsed(TimeUnit.MILLISECONDS), t2.elapsed(TimeUnit.MILLISECONDS), t3.elapsed(TimeUnit.MILLISECONDS), t4.elapsed(TimeUnit.MILLISECONDS)));
Class<?> c = classLoader.findClass(materializedClassName);
if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
return (T) c.newInstance();
} else {
throw new ClassTransformationException("The requested class did not implement the expected interface.");
}
-
+
} catch (CompileException | IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
throw new ClassTransformationException(String.format(
"Failure generating transformation classes for value: \n %s", entireClass), e);
@@ -213,9 +216,7 @@ public class ClassTransformer {
@Override
public void visitInnerClass(String name, String outerName, String innerName, int access) {
- logger.debug(String.format(
- "[Inner Class] Name: %s, outerName: %s, innerName: %s, templateName: %s, newName: %s.", name, outerName,
- innerName, templateName, newName));
+// logger.debug(String.format("[Inner Class] Name: %s, outerName: %s, innerName: %s, templateName: %s, newName: %s.", name, outerName, innerName, templateName, newName));
if (name.startsWith(templateName)) {
super.visitInnerClass(name.replace(templateName, newName), newName, innerName, access);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
index bd23844..a15d34d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
@@ -17,48 +17,38 @@
******************************************************************************/
package org.apache.drill.exec.compile;
-import org.apache.drill.exec.compile.sig.CodeGeneratorSignature;
-import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature;
import org.apache.drill.exec.compile.sig.SignatureHolder;
public class TemplateClassDefinition<T>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TemplateClassDefinition.class);
- private final Class<T> externalInterface;
- private final String templateClassName;
- private final Class<?> internalInterface;
+ private final Class<T> iface;
+ private final Class<?> template;
private final SignatureHolder signature;
- public TemplateClassDefinition(Class<T> externalInterface, String templateClassName, Class<?> internalInterface) {
- this(externalInterface, templateClassName, internalInterface, DefaultGeneratorSignature.class);
- }
- public <X extends CodeGeneratorSignature> TemplateClassDefinition(Class<T> externalInterface, String templateClassName, Class<?> internalInterface, Class<X> signature) {
+ public <X extends T> TemplateClassDefinition(Class<T> iface, Class<X> template) {
super();
- this.externalInterface = externalInterface;
- this.templateClassName = templateClassName;
- this.internalInterface = internalInterface;
+ this.iface = iface;
+ this.template = template;
SignatureHolder holder = null;
try{
- holder = new SignatureHolder(signature);
+ holder = new SignatureHolder(template);
}catch(Exception ex){
- logger.error("Failure while trying to build signature holder for signature. {}", signature.getSimpleName(), ex);
+ logger.error("Failure while trying to build signature holder for signature. {}", template.getName(), ex);
}
this.signature = holder;
}
public Class<T> getExternalInterface() {
- return externalInterface;
- }
-
- public Class<?> getInternalInterface() {
- return internalInterface;
+ return iface;
}
+
public String getTemplateClassName() {
- return templateClassName;
+ return template.getName();
}
public SignatureHolder getSignature(){
@@ -67,9 +57,9 @@ public class TemplateClassDefinition<T>{
@Override
public String toString() {
- return "TemplateClassDefinition [externalInterface=" + externalInterface + ", templateClassName="
- + templateClassName + ", internalInterface=" + internalInterface + ", signature=" + signature + "]";
+ return "TemplateClassDefinition [template=" + template + ", signature=" + signature + "]";
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java
deleted file mode 100644
index dc56206..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.apache.drill.exec.compile.sig;
-
-import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
-
-import javax.inject.Named;
-
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.record.RecordBatch;
-
-public interface DefaultGeneratorSignature extends CodeGeneratorSignature{
-
- public static final GeneratorMapping DEFAULT_SCALAR_MAP = GM("doSetup", "doEval", null, null);
- public static final GeneratorMapping DEFAULT_CONSTANT_MAP = GM("doSetup", "doSetup", null, null);
-
- public static final MappingSet DEFAULT_MAPPING = new MappingSet("inIndex", "outIndex", DEFAULT_SCALAR_MAP, DEFAULT_SCALAR_MAP);
-
- public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing);
- public void doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex);
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
index 0633446..8646b9b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
@@ -1,5 +1,9 @@
package org.apache.drill.exec.compile.sig;
+import org.apache.drill.exec.expr.CodeGenerator.BlockType;
+
+import com.google.common.base.Preconditions;
+
public class GeneratorMapping {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GeneratorMapping.class);
@@ -17,24 +21,31 @@ public class GeneratorMapping {
}
public static GeneratorMapping GM(String setup, String eval, String reset, String cleanup){
- return new GeneratorMapping(setup, eval, reset, cleanup);
+ return create(setup, eval, reset, cleanup);
}
- public String getSetup() {
- return setup;
- }
-
- public String getEval() {
- return eval;
- }
-
- public String getReset() {
- return reset;
+ public static GeneratorMapping create(String setup, String eval, String reset, String cleanup){
+ return new GeneratorMapping(setup, eval, reset, cleanup);
}
-
- public String getCleanup() {
- return cleanup;
+
+ public String getMethodName(BlockType type){
+ switch(type){
+ case CLEANUP:
+ Preconditions.checkNotNull(cleanup, "The current mapping does not have a cleanup method defined.");
+ return cleanup;
+ case EVAL:
+ Preconditions.checkNotNull(eval, "The current mapping does not have an eval method defined.");
+ return eval;
+ case RESET:
+ Preconditions.checkNotNull(reset, "The current mapping does not have a cleanup method defined.");
+ return reset;
+ case SETUP:
+ Preconditions.checkNotNull(setup, "The current mapping does not have a setup method defined.");
+ return setup;
+ default:
+ throw new IllegalStateException();
+ }
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
index e6dcf4e..b95c55c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
@@ -2,6 +2,8 @@ package org.apache.drill.exec.compile.sig;
import java.util.Arrays;
+import org.apache.drill.exec.expr.DirectExpression;
+
import com.google.common.base.Preconditions;
import com.sun.codemodel.JExpr;
import com.sun.codemodel.JExpression;
@@ -14,13 +16,29 @@ public class MappingSet {
private GeneratorMapping[] mappings;
private int mappingIndex;
private GeneratorMapping current;
- private JExpression readIndex;
- private JExpression writeIndex;
+ private DirectExpression readIndex;
+ private DirectExpression writeIndex;
+ private DirectExpression incoming;
+ private DirectExpression outgoing;
+
+ public MappingSet(GeneratorMapping mapping) {
+ this("inIndex", "outIndex", new GeneratorMapping[]{mapping, mapping});
+ }
+ public MappingSet(GeneratorMapping... mappings) {
+ this("inIndex", "outIndex", mappings);
+ }
+
public MappingSet(String readIndex, String writeIndex, GeneratorMapping... mappings) {
- this.readIndex = JExpr.direct(readIndex);
- this.writeIndex = JExpr.direct(writeIndex);
+ this(readIndex, writeIndex, "incoming", "outgoing", mappings);
+ }
+
+ public MappingSet(String readIndex, String writeIndex, String incoming, String outgoing, GeneratorMapping... mappings) {
+ this.readIndex = DirectExpression.direct(readIndex);
+ this.writeIndex = DirectExpression.direct(writeIndex);
+ this.incoming = DirectExpression.direct(incoming);
+ this.outgoing = DirectExpression.direct(outgoing);
Preconditions.checkArgument(mappings.length >= 2);
this.constant = mappings[0];
this.mappings = Arrays.copyOfRange(mappings, 1, mappings.length);
@@ -49,19 +67,26 @@ public class MappingSet {
assert current == mappings[mappingIndex];
mappingIndex--;
if(mappingIndex < 0) throw new IllegalStateException("You tried to traverse higher than the provided mapping provides.");
+ current = mappings[mappingIndex];
}
public GeneratorMapping getCurrentMapping(){
return current;
}
- public JExpression getValueWriteIndex(){
+ public DirectExpression getValueWriteIndex(){
return writeIndex;
}
- public JExpression getValueReadIndex(){
+ public DirectExpression getValueReadIndex(){
return readIndex;
}
+ public DirectExpression getOutgoing(){
+ return outgoing;
+ }
+ public DirectExpression getIncoming(){
+ return incoming;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
index 68b2498..8f79983 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java
@@ -1,10 +1,13 @@
package org.apache.drill.exec.compile.sig;
import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
import java.util.Arrays;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import com.beust.jcommander.internal.Lists;
import com.beust.jcommander.internal.Maps;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
@@ -15,18 +18,31 @@ public class SignatureHolder implements Iterable<CodeGeneratorMethod>{
private final CodeGeneratorMethod[] methods;
private final Map<String, Integer> methodMap;
- public <T extends CodeGeneratorSignature> SignatureHolder(Class<T> signature){
+ public SignatureHolder(Class<?> signature){
Method[] reflectMethods = signature.getDeclaredMethods();
- methods = new CodeGeneratorMethod[reflectMethods.length];
Map<String, Integer> newMap = Maps.newHashMap();
- for(int i =0; i < methods.length; i++){
- methods[i] = new CodeGeneratorMethod(reflectMethods[i]);
+ List<CodeGeneratorMethod> methodHolders = Lists.newArrayList();
+ for(Method m : reflectMethods){
+ if( (m.getModifiers() & Modifier.ABSTRACT) == 0) continue;
+ methodHolders.add(new CodeGeneratorMethod(m));
+ }
+
+ methods = new CodeGeneratorMethod[methodHolders.size()];
+ for(int i =0; i < methodHolders.size(); i++){
+ methods[i] = methodHolders.get(i);
newMap.put(methods[i].getMethodName(), i);
}
+
methodMap = ImmutableMap.copyOf(newMap);
+
+
+ }
+
+ public CodeGeneratorMethod get(int i){
+ return methods[i];
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
index d2e4cf3..0c9ca7e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
@@ -1,6 +1,10 @@
package org.apache.drill.exec.expr;
+import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
+
import java.io.IOException;
+import java.util.LinkedList;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.common.expression.LogicalExpression;
@@ -10,7 +14,6 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.compile.TemplateClassDefinition;
import org.apache.drill.exec.compile.sig.CodeGeneratorArgument;
import org.apache.drill.exec.compile.sig.CodeGeneratorMethod;
-import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.compile.sig.SignatureHolder;
@@ -19,6 +22,8 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.vector.TypeHelper;
+import com.beust.jcommander.internal.Lists;
+import com.beust.jcommander.internal.Maps;
import com.google.common.base.Preconditions;
import com.sun.codemodel.JBlock;
import com.sun.codemodel.JClass;
@@ -34,11 +39,16 @@ import com.sun.codemodel.JVar;
public class CodeGenerator<T>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGenerator.class);
+ public static final GeneratorMapping DEFAULT_SCALAR_MAP = GM("doSetup", "doEval", null, null);
+ public static final GeneratorMapping DEFAULT_CONSTANT_MAP = GM("doSetup", "doSetup", null, null);
+ public static final MappingSet DEFAULT_MAPPING = new MappingSet("inIndex", "outIndex", DEFAULT_SCALAR_MAP, DEFAULT_SCALAR_MAP);
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGenerator.class);
+ public static enum BlockType {SETUP, EVAL, RESET, CLEANUP};
+
public JDefinedClass clazz;
- private JBlock[] parentBlocks;
- private JBlock[] childBlocks;
+ private LinkedList<JBlock>[] blocks;
private final TemplateClassDefinition<T> definition;
private JCodeModel model;
private int index = 0;
@@ -49,12 +59,14 @@ public class CodeGenerator<T>{
private final SignatureHolder sig;
private MappingSet mappings;
private final EvaluationVisitor evaluationVisitor;
+ private final Map<ValueVectorSetup, JVar> vvDeclaration = Maps.newHashMap();
public CodeGenerator(TemplateClassDefinition<T> definition, FunctionImplementationRegistry funcRegistry) {
- this(DefaultGeneratorSignature.DEFAULT_MAPPING, definition, funcRegistry);
+ this(DEFAULT_MAPPING, definition, funcRegistry);
}
+ @SuppressWarnings("unchecked")
public CodeGenerator(MappingSet mappingSet, TemplateClassDefinition<T> definition, FunctionImplementationRegistry funcRegistry) {
Preconditions.checkNotNull(definition.getSignature(), "The signature for defintion %s was incorrectly initialized.", definition);
this.sig = definition.getSignature();
@@ -65,12 +77,10 @@ public class CodeGenerator<T>{
this.definition = definition;
this.model = new JCodeModel();
this.clazz = model._package(packageName)._class(className);
- clazz._implements(definition.getInternalInterface());
- parentBlocks = new JBlock[sig.size()];
+ blocks = (LinkedList<JBlock>[]) new LinkedList[sig.size()];
for(int i =0; i < sig.size(); i++){
- parentBlocks[i] = new JBlock(false, false);
+ blocks[i] = Lists.newLinkedList();
}
- childBlocks = new JBlock[sig.size()];
this.evaluationVisitor = new EvaluationVisitor(funcRegistry);
rotateBlock();
} catch (JClassAlreadyExistsException e) {
@@ -90,26 +100,38 @@ public class CodeGenerator<T>{
return mappings.getCurrentMapping();
}
- private JBlock getBlock(String methodName){
- JBlock blk = this.childBlocks[sig.get(methodName)];
+ public JBlock getBlock(String methodName){
+ JBlock blk = this.blocks[sig.get(methodName)].getLast();
Preconditions.checkNotNull(blk, "Requested method name of %s was not available for signature %s.", methodName, this.sig);
return blk;
}
+ public JBlock getBlock(BlockType type){
+ return getBlock(getCurrentMapping().getMethodName(type));
+ }
+
public JBlock getSetupBlock(){
- return getBlock(getCurrentMapping().getSetup());
+ return getBlock(getCurrentMapping().getMethodName(BlockType.SETUP));
}
public JBlock getEvalBlock(){
- return getBlock(getCurrentMapping().getEval());
+ return getBlock(getCurrentMapping().getMethodName(BlockType.EVAL));
+ }
+ public JBlock getResetBlock(){
+ return getBlock(getCurrentMapping().getMethodName(BlockType.RESET));
+ }
+ public JBlock getCleanupBlock(){
+ return getBlock(getCurrentMapping().getMethodName(BlockType.CLEANUP));
}
-// public JBlock getResetBlock(){
-// return getBlock(getCurrentMapping().getReset());
-// }
-// public JBlock getCleanupBlock(){
-// return getBlock(getCurrentMapping().getCleanup());
-// }
public JVar declareVectorValueSetupAndMember(String batchName, TypedFieldId fieldId){
+ return declareVectorValueSetupAndMember( DirectExpression.direct(batchName), fieldId);
+ }
+
+ public JVar declareVectorValueSetupAndMember(DirectExpression batchName, TypedFieldId fieldId){
+ final ValueVectorSetup setup = new ValueVectorSetup(batchName, fieldId);
+ JVar var = this.vvDeclaration.get(setup);
+ if(var != null) return var;
+
Class<?> valueVectorClass = TypeHelper.getValueVectorClass(fieldId.getType().getMinorType(), fieldId.getType().getMode());
JClass vvClass = model.ref(valueVectorClass);
JClass retClass = vvClass;
@@ -126,16 +148,18 @@ public class CodeGenerator<T>{
JVar obj = b.decl( //
objClass, //
getNextVar("tmp"), //
- JExpr.direct(batchName)
+ batchName
.invoke("getValueAccessorById") //
.arg(JExpr.lit(fieldId.getFieldId())) //
.arg( vvClass.dotclass())
.invoke(vectorAccess)//
);
- b._if(obj.eq(JExpr._null()))._then()._throw(JExpr._new(t).arg(JExpr.lit(String.format("Failure while loading vector %s with id: %s.", vv.name(), fieldId.toString()))));
- //b.assign(vv, JExpr.cast(retClass, ((JExpression) JExpr.cast(wrapperClass, obj) ).invoke(vectorAccess)));
- b.assign(vv, JExpr.cast(retClass, obj ));
+
+ b._if(obj.eq(JExpr._null()))._then()._throw(JExpr._new(t).arg(JExpr.lit(String.format("Failure while loading vector %s with id: %s.", vv.name(), fieldId.toString()))));
+ //b.assign(vv, JExpr.cast(retClass, ((JExpression) JExpr.cast(wrapperClass, obj) ).invoke(vectorAccess)));
+ b.assign(vv, JExpr.cast(retClass, obj ));
+ vvDeclaration.put(setup, vv);
return vv;
}
@@ -145,16 +169,14 @@ public class CodeGenerator<T>{
}
public HoldingContainer addExpr(LogicalExpression ex, boolean rotate){
- logger.debug("Adding next write {}", ex);
+// logger.debug("Adding next write {}", ex);
if(rotate) rotateBlock();
return evaluationVisitor.addExpr(ex, this);
}
public void rotateBlock(){
- for(int i =0; i < childBlocks.length; i++){
- this.childBlocks[i] = new JBlock(true, true);
-
- this.parentBlocks[i].add(childBlocks[i]);
+ for(LinkedList<JBlock> b : blocks){
+ b.add(new JBlock(true, true));
}
}
@@ -176,9 +198,12 @@ public class CodeGenerator<T>{
for(Class<?> c : method.getThrowsIterable()){
m._throws(model.ref(c));
}
-
m._throws(SchemaChangeException.class);
- m.body().add(this.parentBlocks[i++]);
+
+ for(JBlock b : blocks[i++]){
+ if(!b.isEmpty()) m.body().add(b);
+ }
+
}
SingleClassStringWriter w = new SingleClassStringWriter();
@@ -202,6 +227,7 @@ public class CodeGenerator<T>{
public JVar declareClassField(String prefix, JType t){
return clazz.field(JMod.NONE, t, prefix + index++);
}
+
public HoldingContainer declare(MajorType t){
return declare(t, true);
@@ -223,6 +249,50 @@ public class CodeGenerator<T>{
return new HoldingContainer(t, var, var.ref("value"), outputSet);
}
+ private static class ValueVectorSetup{
+ final DirectExpression batch;
+ final TypedFieldId fieldId;
+
+ public ValueVectorSetup(DirectExpression batch, TypedFieldId fieldId) {
+ super();
+ this.batch = batch;
+ this.fieldId = fieldId;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((batch == null) ? 0 : batch.hashCode());
+ result = prime * result + ((fieldId == null) ? 0 : fieldId.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ValueVectorSetup other = (ValueVectorSetup) obj;
+ if (batch == null) {
+ if (other.batch != null)
+ return false;
+ } else if (!batch.equals(other.batch))
+ return false;
+ if (fieldId == null) {
+ if (other.fieldId != null)
+ return false;
+ } else if (!fieldId.equals(other.fieldId))
+ return false;
+ return true;
+ }
+
+
+ }
+
public static class HoldingContainer{
private final JVar holder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DirectExpression.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DirectExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DirectExpression.java
new file mode 100644
index 0000000..ed8dfeb
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DirectExpression.java
@@ -0,0 +1,50 @@
+package org.apache.drill.exec.expr;
+
+import com.sun.codemodel.JExpressionImpl;
+import com.sun.codemodel.JFormatter;
+
+public class DirectExpression extends JExpressionImpl{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DirectExpression.class);
+
+ final String source;
+
+ private DirectExpression(final String source) {
+ super();
+ this.source = source;
+ }
+
+ public void generate( JFormatter f ) {
+ f.p('(').p(source).p(')');
+ }
+
+ public static DirectExpression direct( final String source ) {
+ return new DirectExpression(source);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((source == null) ? 0 : source.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ DirectExpression other = (DirectExpression) obj;
+ if (source == null) {
+ if (other.source != null)
+ return false;
+ } else if (!source.equals(other.source))
+ return false;
+ return true;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggFunc.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggFunc.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggFunc.java
new file mode 100644
index 0000000..d8a3296
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggFunc.java
@@ -0,0 +1,10 @@
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.exec.record.RecordBatch;
+
+public interface DrillAggFunc extends DrillFunc{
+ public void setup(RecordBatch incoming);
+ public void add();
+ public void output();
+ public void reset();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggrFunc.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggrFunc.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggrFunc.java
deleted file mode 100644
index d4433e4..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillAggrFunc.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package org.apache.drill.exec.expr;
-
-import org.apache.drill.exec.record.RecordBatch;
-
-public interface DrillAggrFunc {
- public void setup(RecordBatch incoming);
- public void add();
- public void eval();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
deleted file mode 100644
index 5042708..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.drill.exec.expr;
-
-import org.apache.drill.exec.record.RecordBatch;
-
-public interface DrillBatchFunc {
- public void setup(RecordBatch incoming);
- public void eval();
- public void batchReset();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFunc.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFunc.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFunc.java
index 83c5ed9..81f55b3 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFunc.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFunc.java
@@ -1,9 +1,4 @@
package org.apache.drill.exec.expr;
-import org.apache.drill.exec.record.RecordBatch;
-
public interface DrillFunc {
- public void setup(RecordBatch incoming);
- public void eval();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillSimpleFunc.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillSimpleFunc.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillSimpleFunc.java
new file mode 100644
index 0000000..ef9c7cf
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillSimpleFunc.java
@@ -0,0 +1,8 @@
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.exec.record.RecordBatch;
+
+public interface DrillSimpleFunc extends DrillFunc{
+ public void setup(RecordBatch incoming);
+ public void eval();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 0e66fd0..fd454a7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -1,5 +1,6 @@
package org.apache.drill.exec.expr;
+import java.util.Collections;
import java.util.Set;
import org.apache.drill.common.expression.FunctionCall;
@@ -12,13 +13,14 @@ import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
-import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
-import org.apache.drill.exec.expr.fn.FunctionHolder;
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
+import org.apache.drill.exec.record.NullExpression;
import com.sun.codemodel.JBlock;
import com.sun.codemodel.JConditional;
@@ -37,7 +39,9 @@ public class EvaluationVisitor {
}
public HoldingContainer addExpr(LogicalExpression e, CodeGenerator<?> generator){
- return e.accept(new ConstantFilter(ConstantExpressionIdentifier.getConstantExpressionSet(e)), generator);
+// Set<LogicalExpression> constantBoundaries = ConstantExpressionIdentifier.getConstantExpressionSet(e);
+ Set<LogicalExpression> constantBoundaries = Collections.emptySet();
+ return e.accept(new ConstantFilter(constantBoundaries), generator);
}
@@ -46,12 +50,18 @@ public class EvaluationVisitor {
@Override
public HoldingContainer visitFunctionCall(FunctionCall call, CodeGenerator<?> generator) throws RuntimeException {
+ DrillFuncHolder holder = registry.getFunction(call);
+ JVar[] workspaceVars = holder.renderStart(generator, null);
+
+
+ if(holder.isNested()) generator.getMappingSet().enterChild();
HoldingContainer[] args = new HoldingContainer[call.args.size()];
for (int i = 0; i < call.args.size(); i++) {
args[i] = call.args.get(i).accept(this, generator);
}
- FunctionHolder holder = registry.getFunction(call);
- return holder.renderFunction(generator, args);
+ holder.renderMiddle(generator, args, workspaceVars);
+ if(holder.isNested()) generator.getMappingSet().exitChild();
+ return holder.renderEnd(generator, args, workspaceVars);
}
@Override
@@ -104,6 +114,7 @@ public class EvaluationVisitor {
return output;
}
+
@Override
public HoldingContainer visitSchemaPath(SchemaPath path, CodeGenerator<?> generator) throws RuntimeException {
throw new UnsupportedOperationException("All schema paths should have been replaced with ValueVectorExpressions.");
@@ -141,6 +152,8 @@ public class EvaluationVisitor {
return visitReturnValueExpression((ReturnValueExpression) e, generator);
}else if(e instanceof HoldingContainerExpression){
return ((HoldingContainerExpression) e).getContainer();
+ }else if(e instanceof NullExpression){
+ return generator.declare(Types.optional(MinorType.INT));
} else {
return super.visitUnknown(e, generator);
}
@@ -148,22 +161,40 @@ public class EvaluationVisitor {
}
private HoldingContainer visitValueVectorWriteExpression(ValueVectorWriteExpression e, CodeGenerator<?> generator) {
+
LogicalExpression child = e.getChild();
- HoldingContainer hc = child.accept(this, generator);
+ HoldingContainer inputContainer = child.accept(this, generator);
JBlock block = generator.getEvalBlock();
JExpression outIndex = generator.getMappingSet().getValueWriteIndex();
- JVar vv = generator.declareVectorValueSetupAndMember("outgoing", e.getFieldId());
-
- if (hc.isOptional()) {
- vv.invoke("getMutator").invoke("set").arg(outIndex);
- JConditional jc = block._if(hc.getIsSet().eq(JExpr.lit(0)).not());
- block = jc._then();
+ JVar vv = generator.declareVectorValueSetupAndMember(generator.getMappingSet().getOutgoing(), e.getFieldId());
+ String setMethod = e.isSafe() ? "setSafe" : "set";
+
+ JInvocation setMeth;
+ if (Types.usesHolderForGet(inputContainer.getMajorType())) {
+ setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getHolder());
+ }else{
+ setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getValue());
}
- if (hc.getMinorType() == TypeProtos.MinorType.VARCHAR || hc.getMinorType() == TypeProtos.MinorType.VARBINARY) {
- block.add(vv.invoke("getMutator").invoke("set").arg(outIndex).arg(hc.getHolder()));
- } else {
- block.add(vv.invoke("getMutator").invoke("set").arg(outIndex).arg(hc.getValue()));
+
+ if(e.isSafe()){
+ HoldingContainer outputContainer = generator.declare(Types.REQUIRED_BIT);
+ block.assign(outputContainer.getValue(), JExpr.lit(1));
+ if(inputContainer.isOptional()){
+// block._if(vv.invoke("getMutator").invoke(setMethod).arg(outIndex).not())._then().assign(outputContainer.getValue(), JExpr.lit(0));
+ JConditional jc = block._if(inputContainer.getIsSet().eq(JExpr.lit(0)).not());
+ block = jc._then();
+ }
+ block._if(setMeth.not())._then().assign(outputContainer.getValue(), JExpr.lit(0));
+ return outputContainer;
+ }else{
+ if (inputContainer.isOptional()) {
+// block.add(vv.invoke("getMutator").invoke(setMethod).arg(outIndex));
+ JConditional jc = block._if(inputContainer.getIsSet().eq(JExpr.lit(0)).not());
+ block = jc._then();
+ }
+ block.add(setMeth);
}
+
return null;
}
@@ -171,7 +202,7 @@ public class EvaluationVisitor {
throws RuntimeException {
// declare value vector
- JVar vv1 = generator.declareVectorValueSetupAndMember("incoming", e.getFieldId());
+ JVar vv1 = generator.declareVectorValueSetupAndMember(generator.getMappingSet().getIncoming(), e.getFieldId());
JExpression indexVariable = generator.getMappingSet().getValueReadIndex();
JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get");
@@ -306,6 +337,7 @@ public class EvaluationVisitor {
}
}
+
@Override
public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator<?> generator) throws RuntimeException {
if (constantBoundaries.contains(e)) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index cb2be68..0a5dcdd 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -96,9 +96,10 @@ public class ExpressionTreeMaterializer {
@Override
public LogicalExpression visitSchemaPath(SchemaPath path) {
- logger.debug("Visiting schema path {}", path);
+// logger.debug("Visiting schema path {}", path);
TypedFieldId tfId = batch.getValueVectorId(path);
if (tfId == null) {
+ logger.warn("Unable to find value vector of path {}, returning null instance.", path);
return NullExpression.INSTANCE;
} else {
return new ValueVectorReadExpression(tfId);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java
index 6848234..74dcc0d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java
@@ -16,25 +16,12 @@ public class ValueVectorReadExpression implements LogicalExpression{
private final MajorType type;
private final TypedFieldId fieldId;
private final boolean superReader;
- private final String indexVariableName;
- public ValueVectorReadExpression(TypedFieldId tfId){
- this(tfId, null);
- }
- public ValueVectorReadExpression(TypedFieldId tfId, String indexVariableName) {
+ public ValueVectorReadExpression(TypedFieldId tfId){
this.type = tfId.getType();
this.fieldId = tfId;
this.superReader = tfId.isHyperReader();
- this.indexVariableName = indexVariableName;
- }
-
- public boolean isNamed(){
- return indexVariableName != null;
- }
-
- public String getIndexVariableName(){
- return indexVariableName;
}
public TypedFieldId getTypedFieldId(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java
index 7f2d847..bfecb18 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java
@@ -16,16 +16,16 @@ public class ValueVectorWriteExpression implements LogicalExpression {
private final TypedFieldId fieldId;
private final LogicalExpression child;
- private final String indexVariableName;
+ private final boolean safe;
public ValueVectorWriteExpression(TypedFieldId fieldId, LogicalExpression child){
- this(fieldId, child, null);
+ this(fieldId, child, false);
}
- public ValueVectorWriteExpression(TypedFieldId fieldId, LogicalExpression child, String indexVariableName){
+ public ValueVectorWriteExpression(TypedFieldId fieldId, LogicalExpression child, boolean safe){
this.fieldId = fieldId;
this.child = child;
- this.indexVariableName = indexVariableName;
+ this.safe = safe;
}
public TypedFieldId getFieldId() {
@@ -37,6 +37,11 @@ public class ValueVectorWriteExpression implements LogicalExpression {
return Types.NULL;
}
+
+ public boolean isSafe() {
+ return safe;
+ }
+
@Override
public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
return visitor.visitUnknown(this, value);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
index a77b4b0..d702b20 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
@@ -4,33 +4,72 @@ import java.util.List;
import java.util.Map;
import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.CodeGenerator.BlockType;
import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
-public class DrillAggFuncHolder extends DrillFuncHolder{
+import com.google.common.base.Preconditions;
+import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JMod;
+import com.sun.codemodel.JVar;
+
+class DrillAggFuncHolder extends DrillFuncHolder{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillAggFuncHolder.class);
+ private final String setup;
+ private final String reset;
+ private final String add;
+ private final String output;
+ private final String cleanup;
public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative,
String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
Map<String, String> methods, List<String> imports) {
super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports);
-
+ Preconditions.checkArgument(nullHandling == NullHandling.INTERNAL, "An aggregation function is required to do its own null handling.");
+ setup = methods.get("setup");
+ reset = methods.get("reset");
+ add = methods.get("add");
+ output = methods.get("output");
+ cleanup = methods.get("cleanup");
+ Preconditions.checkNotNull(add);
+ Preconditions.checkNotNull(output);
+ Preconditions.checkNotNull(reset);
}
- public HoldingContainer renderFunction(CodeGenerator<?> g, HoldingContainer[] inputVariables){
- g.getMappingSet().enterChild();
- renderInside(g, inputVariables);
- g.getMappingSet().exitChild();
- return renderOutside(g);
+ public boolean isNested(){
+ return true;
}
- private void renderInside(CodeGenerator<?> g, HoldingContainer[] inputVariables){
-
+ @Override
+ public JVar[] renderStart(CodeGenerator<?> g, HoldingContainer[] inputVariables) {
+ JVar[] workspaceJVars = declareWorkspaceVariables(g);
+ generateBody(g, BlockType.SETUP, setup, workspaceJVars);
+ return workspaceJVars;
}
-
- private HoldingContainer renderOutside(CodeGenerator<?> g){
- return null;
+
+
+ @Override
+ public void renderMiddle(CodeGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+ addProtectedBlock(g, g.getBlock(BlockType.EVAL), add, inputVariables, workspaceJVars);
+ }
+
+
+ @Override
+ public HoldingContainer renderEnd(CodeGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+ HoldingContainer out = g.declare(returnValue.type, false);
+ JBlock sub = new JBlock();
+ g.getEvalBlock().add(sub);
+ JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type)));
+ addProtectedBlock(g, sub, output, null, workspaceJVars);
+ sub.assign(out.getHolder(), internalOutput);
+
+ generateBody(g, BlockType.RESET, reset, workspaceJVars);
+ generateBody(g, BlockType.CLEANUP, cleanup, workspaceJVars);
+ return out;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index 8e69609..91f7705 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -3,100 +3,173 @@ package org.apache.drill.exec.expr.fn;
import java.util.List;
import java.util.Map;
-import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.CodeGenerator.BlockType;
import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.sun.codemodel.JBlock;
-import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JExpression;
import com.sun.codemodel.JMod;
import com.sun.codemodel.JVar;
-public class DrillFuncHolder extends FunctionHolder{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolder.class);
-
- private final String setupBody;
- private final String evalBody;
+public abstract class DrillFuncHolder {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
+ protected final FunctionTemplate.FunctionScope scope;
+ protected final FunctionTemplate.NullHandling nullHandling;
+ protected final boolean isBinaryCommutative;
+ protected final String functionName;
+ protected final ImmutableList<String> imports;
+ protected final WorkspaceReference[] workspaceVars;
+ protected final ValueReference[] parameters;
+ protected final ValueReference returnValue;
+ protected final ImmutableMap<String, String> methodMap;
- public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative,
- String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
- Map<String, String> methods, List<String> imports) {
- super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports);
- setupBody = methods.get("setup");
- evalBody = methods.get("eval");
- Preconditions.checkNotNull(evalBody);
+ public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) {
+ super();
+ this.scope = scope;
+ this.nullHandling = nullHandling;
+ this.workspaceVars = workspaceVars;
+ this.isBinaryCommutative = isBinaryCommutative;
+ this.functionName = functionName;
+ this.methodMap = ImmutableMap.copyOf(methods);
+ this.parameters = parameters;
+ this.returnValue = returnValue;
+ this.imports = ImmutableList.copyOf(imports);
}
-
- private void generateSetupBody(CodeGenerator<?> g){
- if(!Strings.isNullOrEmpty(setupBody)){
- JBlock sub = new JBlock(true, true);
- addProtectedBlock(g, sub, setupBody, null);
- g.getSetupBlock().directStatement(String.format("/** start setup for function %s **/ ", functionName));
- g.getSetupBlock().add(sub);
- g.getSetupBlock().directStatement(String.format("/** end setup for function %s **/ ", functionName));
- }
+
+ public List<String> getImports() {
+ return imports;
}
+
+ public JVar[] renderStart(CodeGenerator<?> g, HoldingContainer[] inputVariables){
+ return declareWorkspaceVariables(g);
+ };
- public HoldingContainer renderFunction(CodeGenerator<?> g, HoldingContainer[] inputVariables){
- generateSetupBody(g);
- return generateEvalBody(g, inputVariables);
- }
+ public void renderMiddle(CodeGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars){};
+ public abstract HoldingContainer renderEnd(CodeGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars);
+ public abstract boolean isNested();
- private HoldingContainer generateEvalBody(CodeGenerator<?> g, HoldingContainer[] inputVariables){
-
- //g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName));
-
- JBlock sub = new JBlock(true, true);
- JBlock topSub = sub;
- HoldingContainer out = null;
- MajorType returnValueType = returnValue.type;
-
- // add outside null handling if it is defined.
- if(nullHandling == NullHandling.NULL_IF_NULL){
- JExpression e = null;
- for(HoldingContainer v : inputVariables){
- if(v.isOptional()){
- if(e == null){
- e = v.getIsSet();
- }else{
- e = e.mul(v.getIsSet());
- }
- }
- }
-
- if(e != null){
- // if at least one expression must be checked, set up the conditional.
- returnValueType = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build();
- out = g.declare(returnValueType);
- e = e.eq(JExpr.lit(0));
- JConditional jc = sub._if(e);
- jc._then().assign(out.getIsSet(), JExpr.lit(0));
- sub = jc._else();
+ protected JVar[] declareWorkspaceVariables(CodeGenerator<?> g){
+ JVar[] workspaceJVars = new JVar[workspaceVars.length];
+ for(int i =0 ; i < workspaceVars.length; i++){
+ workspaceJVars[i] = g.declareClassField("work", g.getModel()._ref(workspaceVars[i].type));
+ }
+ return workspaceJVars;
+ }
+
+ protected void generateBody(CodeGenerator<?> g, BlockType bt, String body, JVar[] workspaceJVars){
+ if(!Strings.isNullOrEmpty(body) && !body.trim().isEmpty()){
+ JBlock sub = new JBlock(true, true);
+ addProtectedBlock(g, sub, body, null, workspaceJVars);
+ g.getBlock(bt).directStatement(String.format("/** start %s for function %s **/ ", bt.name(), functionName));
+ g.getBlock(bt).add(sub);
+ g.getBlock(bt).directStatement(String.format("/** end %s for function %s **/ ", bt.name(), functionName));
+ }
+ }
+
+ protected void addProtectedBlock(CodeGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars){
+
+ if(inputVariables != null){
+ for(int i =0; i < inputVariables.length; i++){
+ ValueReference parameter = parameters[i];
+ HoldingContainer inputVariable = inputVariables[i];
+ sub.decl(inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
}
}
+
+ JVar[] internalVars = new JVar[workspaceJVars.length];
+ for(int i =0; i < workspaceJVars.length; i++){
+ internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
+ }
- if(out == null) out = g.declare(returnValueType);
-
- // add the subblock after the out declaration.
- g.getEvalBlock().add(topSub);
+ Preconditions.checkNotNull(body);
+ sub.directStatement(body);
+ // reassign workspace variables back to global space.
+ for(int i =0; i < workspaceJVars.length; i++){
+ sub.assign(workspaceJVars[i], internalVars[i]);
+ }
+ }
+
+
+
+
+ public boolean matches(FunctionCall call){
+ if(!softCompare(call.getMajorType(), returnValue.type)){
+// logger.debug(String.format("Call [%s] didn't match as return type [%s] was different than expected [%s]. ", call.getDefinition().getName(), returnValue.type, call.getMajorType()));
+ return false;
+ }
+ if(call.args.size() != parameters.length){
+// logger.debug(String.format("Call [%s] didn't match as the number of arguments provided [%d] were different than expected [%d]. ", call.getDefinition().getName(), parameters.length, call.args.size()));
+ return false;
+ }
+ for(int i =0; i < parameters.length; i++){
+ ValueReference param = parameters[i];
+ LogicalExpression arg = call.args.get(i);
+ if(!softCompare(param.type, arg.getMajorType())){
+// logger.debug(String.format("Call [%s] didn't match as the argument [%s] didn't match the expected type [%s]. ", call.getDefinition().getName(), arg.getMajorType(), param.type));
+ return false;
+ }
+ }
- JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValueType), returnValue.name, JExpr._new(g.getHolderType(returnValueType)));
- addProtectedBlock(g, sub, evalBody, inputVariables);
- if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode
- sub.assign(out.getHolder(), internalOutput);
+ return true;
+ }
+
+ private boolean softCompare(MajorType a, MajorType b){
+ return Types.softEquals(a, b, nullHandling == NullHandling.NULL_IF_NULL);
+ }
+
+ public String getFunctionName() {
+ return functionName;
+ }
- return out;
+ public static class ValueReference{
+ MajorType type;
+ String name;
+ public ValueReference(MajorType type, String name) {
+ super();
+ Preconditions.checkNotNull(type);
+ Preconditions.checkNotNull(name);
+ this.type = type;
+ this.name = name;
+ }
+ @Override
+ public String toString() {
+ return "ValueReference [type=" + type + ", name=" + name + "]";
+ }
}
+
+
+ public static class WorkspaceReference{
+ Class<?> type;
+ String name;
+
+
+ public WorkspaceReference(Class<?> type, String name) {
+ super();
+ Preconditions.checkNotNull(type);
+ Preconditions.checkNotNull(name);
+ this.type = type;
+ this.name = name;
+ }
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
new file mode 100644
index 0000000..0a8e35f
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
@@ -0,0 +1,103 @@
+package org.apache.drill.exec.expr.fn;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.CodeGenerator.BlockType;
+import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JMod;
+import com.sun.codemodel.JVar;
+
+class DrillSimpleFuncHolder extends DrillFuncHolder{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSimpleFuncHolder.class);
+
+ private final String setupBody;
+ private final String evalBody;
+ private final String resetBody;
+ private final String cleanupBody;
+
+
+ public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative,
+ String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
+ Map<String, String> methods, List<String> imports) {
+ super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports);
+ setupBody = methods.get("setup");
+ evalBody = methods.get("eval");
+ resetBody = methods.get("reset");
+ cleanupBody = methods.get("cleanup");
+ Preconditions.checkNotNull(evalBody);
+
+ }
+
+ public boolean isNested(){
+ return false;
+ }
+
+ public HoldingContainer renderEnd(CodeGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars){
+ generateBody(g, BlockType.SETUP, setupBody, workspaceJVars);
+ HoldingContainer c = generateEvalBody(g, inputVariables, evalBody, workspaceJVars);
+ generateBody(g, BlockType.RESET, resetBody, workspaceJVars);
+ generateBody(g, BlockType.CLEANUP, cleanupBody, workspaceJVars);
+ return c;
+ }
+
+ protected HoldingContainer generateEvalBody(CodeGenerator<?> g, HoldingContainer[] inputVariables, String body, JVar[] workspaceJVars){
+
+ //g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName));
+
+ JBlock sub = new JBlock(true, true);
+ JBlock topSub = sub;
+ HoldingContainer out = null;
+ MajorType returnValueType = returnValue.type;
+
+ // add outside null handling if it is defined.
+ if(nullHandling == NullHandling.NULL_IF_NULL){
+ JExpression e = null;
+ for(HoldingContainer v : inputVariables){
+ if(v.isOptional()){
+ if(e == null){
+ e = v.getIsSet();
+ }else{
+ e = e.mul(v.getIsSet());
+ }
+ }
+ }
+
+ if(e != null){
+ // if at least one expression must be checked, set up the conditional.
+ returnValueType = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build();
+ out = g.declare(returnValueType);
+ e = e.eq(JExpr.lit(0));
+ JConditional jc = sub._if(e);
+ jc._then().assign(out.getIsSet(), JExpr.lit(0));
+ sub = jc._else();
+ }
+ }
+
+ if(out == null) out = g.declare(returnValueType);
+
+ // add the subblock after the out declaration.
+ g.getEvalBlock().add(topSub);
+
+
+ JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValueType), returnValue.name, JExpr._new(g.getHolderType(returnValueType)));
+ addProtectedBlock(g, sub, body, inputVariables, workspaceJVars);
+ if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode
+ sub.assign(out.getHolder(), internalOutput);
+ if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode
+ return out;
+ }
+
+}