You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/28 02:30:37 UTC
[2/5] DRILL-185: Extend code/clean code generator and add first
aggregate functions.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java
new file mode 100644
index 0000000..1243aa6
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java
@@ -0,0 +1,19 @@
+package org.apache.drill.exec.vector.allocator;
+
+import org.apache.drill.exec.vector.VariableWidthVector;
+
+class VariableVectorAllocator extends VectorAllocator{
+ VariableWidthVector in;
+ VariableWidthVector out;
+
+ public VariableVectorAllocator(VariableWidthVector in, VariableWidthVector out) {
+ super();
+ this.in = in;
+ this.out = out;
+ }
+
+ public void alloc(int recordCount){
+ out.allocateNew(in.getByteCapacity(), recordCount);
+ out.getMutator().setValueCount(recordCount);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
new file mode 100644
index 0000000..3b5187b
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VectorAllocator.java
@@ -0,0 +1,30 @@
+package org.apache.drill.exec.vector.allocator;
+
+import org.apache.drill.exec.vector.FixedWidthVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
+
+public abstract class VectorAllocator{
+ public abstract void alloc(int recordCount);
+
+ public static VectorAllocator getAllocator(ValueVector in, ValueVector outgoing){
+ if(outgoing instanceof FixedWidthVector){
+ return new FixedVectorAllocator((FixedWidthVector) outgoing);
+ }else if(outgoing instanceof VariableWidthVector && in instanceof VariableWidthVector){
+ return new VariableVectorAllocator( (VariableWidthVector) in, (VariableWidthVector) outgoing);
+ }else{
+ throw new UnsupportedOperationException();
+ }
+ }
+
+
+ public static VectorAllocator getAllocator(ValueVector outgoing, int averageBytesPerVariable){
+ if(outgoing instanceof FixedWidthVector){
+ return new FixedVectorAllocator((FixedWidthVector) outgoing);
+ }else if(outgoing instanceof VariableWidthVector){
+ return new VariableEstimatedVector( (VariableWidthVector) outgoing, averageBytesPerVariable);
+ }else{
+ throw new UnsupportedOperationException();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java b/sandbox/prototype/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
index 72322d5..4998888 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/parquet/hadoop/CodecFactoryExposer.java
@@ -17,13 +17,13 @@
******************************************************************************/
package parquet.hadoop;
-import com.sun.corba.se.impl.interceptors.CodecFactoryImpl;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
+
import parquet.bytes.BytesInput;
import parquet.hadoop.metadata.CompressionCodecName;
-import java.io.IOException;
-
public class CodecFactoryExposer{
private CodecFactory codecFactory;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/ExampleInternalInterface.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/ExampleInternalInterface.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/ExampleInternalInterface.java
deleted file mode 100644
index 7ddf103..0000000
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/ExampleInternalInterface.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.compile;
-
-public interface ExampleInternalInterface {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExampleInternalInterface.class);
-
- public String getInternalData();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index 16c993d..7bf623c 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -38,10 +38,7 @@ public class TestClassTransformation {
private void testBasicClassCompilation(boolean useJanino) throws ClassTransformationException{
final String output = "hello world, the time is now " + System.currentTimeMillis();
- TemplateClassDefinition<ExampleExternalInterface> def = new TemplateClassDefinition<ExampleExternalInterface>(
- ExampleExternalInterface.class, "org.apache.drill.exec.compile.ExampleTemplate",
- ExampleInternalInterface.class);
-
+ TemplateClassDefinition<ExampleExternalInterface> def = new TemplateClassDefinition<ExampleExternalInterface>(ExampleExternalInterface.class, ExampleTemplate.class);
ClassTransformer ct = new ClassTransformer();
QueryClassLoader loader = new QueryClassLoader(useJanino);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
index dddd322..6edde0d 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
@@ -4,6 +4,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.ScreenCreator.ScreenRoot;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
@@ -28,6 +29,10 @@ public class SimpleRootExec implements RootExec, Iterable<ValueVector>{
}
+ public FragmentContext getContext(){
+ return incoming.getContext();
+ }
+
public SelectionVector2 getSelectionVector2(){
return incoming.getSelectionVector2();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
new file mode 100644
index 0000000..b13948d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
@@ -0,0 +1,115 @@
+package org.apache.drill.exec.physical.impl.agg;
+
+import static org.junit.Assert.*;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.IntVector;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.yammer.metrics.MetricRegistry;
+
+public class TestAgg {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAgg.class);
+ DrillConfig c = DrillConfig.create();
+
+ private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnection connection, String file) throws Exception{
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+ return exec;
+ }
+
+ @Test
+ public void oneKeyAgg(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable{
+ SimpleRootExec exec = doTest(bitContext, connection, "/agg/test1.json");
+
+ while(exec.next()){
+ BigIntVector cnt = exec.getValueVectorById(new SchemaPath("cnt", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ IntVector key = exec.getValueVectorById(new SchemaPath("blue", ExpressionPosition.UNKNOWN), IntVector.class);
+ long[] cntArr = {10001, 9999};
+ int[] keyArr = {Integer.MIN_VALUE, Integer.MAX_VALUE};
+
+ for(int i =0; i < exec.getRecordCount(); i++){
+ assertEquals(cntArr[i], cnt.getAccessor().getObject(i));
+ assertEquals(keyArr[i], key.getAccessor().getObject(i));
+ }
+ }
+
+ if(exec.getContext().getFailureCause() != null){
+ throw exec.getContext().getFailureCause();
+ }
+ assertTrue(!exec.getContext().isFailed());
+
+ }
+
+ @Test
+ public void twoKeyAgg(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable{
+ SimpleRootExec exec = doTest(bitContext, connection, "/agg/twokey.json");
+
+ while(exec.next()){
+ IntVector key1 = exec.getValueVectorById(new SchemaPath("key1", ExpressionPosition.UNKNOWN), IntVector.class);
+ BigIntVector key2 = exec.getValueVectorById(new SchemaPath("key2", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ BigIntVector cnt = exec.getValueVectorById(new SchemaPath("cnt", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ BigIntVector total = exec.getValueVectorById(new SchemaPath("total", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ int[] keyArr1 = {Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE};
+ long[] keyArr2 = {0,1,2,0,1,2};
+ long[] cntArr = {34,34,34,34,34,34};
+ long[] totalArr = {0,34,68,0,34,68};
+
+ for(int i =0; i < exec.getRecordCount(); i++){
+// System.out.print(key1.getAccessor().getObject(i));
+// System.out.print("\t");
+// System.out.print(key2.getAccessor().getObject(i));
+// System.out.print("\t");
+// System.out.print(cnt.getAccessor().getObject(i));
+// System.out.print("\t");
+// System.out.print(total.getAccessor().getObject(i));
+// System.out.println();
+ assertEquals(cntArr[i], cnt.getAccessor().getObject(i));
+ assertEquals(keyArr1[i], key1.getAccessor().getObject(i));
+ assertEquals(keyArr2[i], key2.getAccessor().getObject(i));
+ assertEquals(totalArr[i], total.getAccessor().getObject(i));
+ }
+ }
+
+ if(exec.getContext().getFailureCause() != null){
+ throw exec.getContext().getFailureCause();
+ }
+ assertTrue(!exec.getContext().isFailed());
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception{
+ // pause to get logger to catch up.
+ Thread.sleep(1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java
deleted file mode 100644
index 4e5e4c7..0000000
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.drill.exec.test.generated;
-
-import org.apache.drill.exec.compile.JaninoClassCompiler;
-
-
-public class TestGeneratedClass {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestGeneratedClass.class);
- public static void main(String[] args) throws Exception{
- String s = "package org.apache.drill.exec.test.generated;\n\nimport org.apache.drill.exec.expr.holders.LongHolder;\nimport org.apache.drill.exec.ops.FragmentContext;\nimport org.apache.drill.exec.record.RecordBatch;\nimport org.apache.drill.exec.record.vector.NullableFixed8;\n\npublic class Test1 {\n\n NullableFixed8 vv0;\n NullableFixed8 vv5;\n NullableFixed8 vv7;\n NullableFixed8 vv12;\n\n public void setupEvaluators(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) {\n {\n {\n Object obj1 = (outgoing).getValueVectorById(1, NullableFixed8 .class);\n vv0 = ((NullableFixed8) obj1);\n Object obj6 = (outgoing).getValueVectorById(0, NullableFixed8 .class);\n vv5 = ((NullableFixed8) obj6);\n }\n {\n Object obj8 = (outgoing).getValueVectorById(1, NullableFixed8 .class);\n vv7 = ((NullableFixed8) obj8);\n Obj
ect obj13 = (outgoing).getValueVectorById(1, NullableFixed8 .class);\n vv12 = ((NullableFixed8) obj13);\n }\n }\n }\n\n public void doPerRecordWork(int inIndex, int outIndex) {\n {\n {\n LongHolder out2 = new LongHolder();\n out2 .value = vv0 .get((inIndex));\n LongHolder out3 = new LongHolder();\n out3 .value = 1L;\n LongHolder out4 = new LongHolder();\n {\n final LongHolder out = new LongHolder();\n final LongHolder left = out2;\n final LongHolder right = out3;\n out.value = left.value + right.value;\n\n out4 = out;\n }\n vv5 .set((outIndex), out4 .value);\n }\n {\n LongHolder out9 = new LongHolder();\n out9 .value = vv7 .get((inIndex));\n Long
Holder out10 = new LongHolder();\n out10 .value = 2L;\n LongHolder out11 = new LongHolder();\n {\n final LongHolder out = new LongHolder();\n final LongHolder left = out9;\n final LongHolder right = out10;\n out.value = left.value + right.value;\n\n out11 = out;\n }\n vv12 .set((outIndex), out11 .value);\n }\n }\n }\n\n}";
- JaninoClassCompiler cc = new JaninoClassCompiler(ClassLoader.getSystemClassLoader());
- byte[] b = cc.getClassByteCode("Test12", s);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/resources/agg/test1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/agg/test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/agg/test1.json
new file mode 100644
index 0000000..eb4647d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/agg/test1.json
@@ -0,0 +1,46 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-sub-scan",
+ entries:[
+ {records: 20000, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"sort",
+ orderings: [
+ {expr: "blue"}
+ ]
+ },
+ {
+ @id:3,
+ child: 2,
+ pop:"streaming-aggregate",
+ keys: [
+ { ref: "blue", expr: "blue" }
+ ],
+ exprs: [
+ { ref: "cnt", expr:"count(1)" }
+ ]
+ },
+ {
+ @id: 4,
+ child: 3,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e43093d9/sandbox/prototype/exec/java-exec/src/test/resources/agg/twokey.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/agg/twokey.json b/sandbox/prototype/exec/java-exec/src/test/resources/agg/twokey.json
new file mode 100644
index 0000000..a237279
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/agg/twokey.json
@@ -0,0 +1,59 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-sub-scan",
+ entries:[
+ {records: 204, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child:1,
+ pop:"project",
+ exprs:[
+ {ref: "key1", expr: "blue"},
+ {ref: "alt", expr: "alternate3()"}
+ ]
+ },
+ {
+ @id:3,
+ child: 2,
+ pop:"sort",
+ orderings: [
+ {expr: "key1"},
+ {expr: "alt"}
+ ]
+ },
+ {
+ @id:4,
+ child: 3,
+ pop:"streaming-aggregate",
+ keys: [
+ { ref: "key1", expr: "key1" },
+ { ref: "key2", expr: "alt" }
+
+ ],
+ exprs: [
+ { ref: "cnt", expr:"count(1)" },
+ { ref: "total", expr:"sum(alt)" }
+ ]
+ },
+ {
+ @id: 5,
+ child: 4,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file