You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/07/20 03:58:06 UTC
[45/53] [abbrv] Working project operator. Update to ASM 4.1
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
index c314cd4..b808dc6 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
@@ -43,4 +43,7 @@ public class SelectionVector2{
return buffer.getChar(directIndex);
}
+ public void setIndex(int directIndex, char value){
+ buffer.setChar(directIndex, value);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index 47aee7d..cb1e1d6 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -37,15 +37,16 @@ public class TestClassTransformation {
private void testBasicClassCompilation(boolean useJanino) throws ClassTransformationException{
final String output = "hello world, the time is now " + System.currentTimeMillis();
- @SuppressWarnings("unchecked")
- TemplateClassDefinition<ExampleExternalInterface, Object> def = new TemplateClassDefinition(
+
+ TemplateClassDefinition<ExampleExternalInterface> def = new TemplateClassDefinition<ExampleExternalInterface>(
ExampleExternalInterface.class, "org.apache.drill.exec.compile.ExampleTemplate",
- ExampleInternalInterface.class, Object.class);
+ ExampleInternalInterface.class, "a", "b");
+
+
ClassTransformer ct = new ClassTransformer();
QueryClassLoader loader = new QueryClassLoader(useJanino);
- ExampleExternalInterface instance = ct.getImplementationClass(loader, def,
- "public String getInternalData(){return \"" + output + "\";}",
- new Object());
+ ExampleExternalInterface instance = ct.getImplementationClassByBody(loader, def,
+ "public String getInternalData(){return \"" + output + "\";}");
System.out.println(String.format("Generated a new class %s that provides the following getData response '%s'.",
instance.getClass().getCanonicalName(), instance.getData()));
assertEquals(instance.getData(), output);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index 16f7802..007abb3 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -23,6 +23,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.physical.impl.project.Projector;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.TypedFieldId;
import org.apache.drill.exec.record.vector.Fixed4;
@@ -48,7 +49,7 @@ public class ExpressionTest {
new NonStrictExpectations() {
{
- batch.getValueVector(new SchemaPath("alpha", ExpressionPosition.UNKNOWN));
+ batch.getValueVectorId(new SchemaPath("alpha", ExpressionPosition.UNKNOWN));
result = tfid;
batch.getValueVectorById(tfid.getFieldId(), Fixed4.class);
result = new Fixed4(null, null);
@@ -65,7 +66,7 @@ public class ExpressionTest {
new Expectations() {
{
- batch.getValueVector(new SchemaPath("alpha", ExpressionPosition.UNKNOWN));
+ batch.getValueVectorId(new SchemaPath("alpha", ExpressionPosition.UNKNOWN));
result = tfid;
// batch.getValueVectorById(tfid); result = new Fixed4(null, null);
}
@@ -95,7 +96,7 @@ public class ExpressionTest {
assertEquals(0, error.getErrorCount());
}
- CodeGenerator cg = new CodeGenerator("setup", "eval", new FunctionImplementationRegistry(DrillConfig.create()));
+ CodeGenerator<Projector> cg = new CodeGenerator<Projector>(Projector.TEMPLATE_DEFINITION, new FunctionImplementationRegistry(DrillConfig.create()));
cg.addNextWrite(new ValueVectorWriteExpression(-1, materializedExpr));
return cg.generate();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
new file mode 100644
index 0000000..093c58f
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
@@ -0,0 +1,40 @@
+package org.apache.drill.exec.physical.impl;
+
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.impl.ScreenCreator.ScreenRoot;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatch.IterOutcome;
+import org.apache.drill.exec.record.RecordBatch.TypedFieldId;
+import org.apache.drill.exec.record.vector.ValueVector;
+
+public class SimpleRootExec implements RootExec{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleRootExec.class);
+
+ private RecordBatch incoming;
+
+ public SimpleRootExec(RootExec e){
+ if(e instanceof ScreenRoot){
+ incoming = ((ScreenRoot)e).getIncoming();
+ }else{
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+
+ public <T extends ValueVector<T>> T getValueVectorById(SchemaPath path, Class<?> vvClass){
+ TypedFieldId tfid = incoming.getValueVectorId(path);
+ return incoming.getValueVectorById(tfid.getFieldId(), vvClass);
+ }
+
+ @Override
+ public boolean next() {
+ return incoming.next() != IterOutcome.NONE;
+ }
+
+ @Override
+ public void stop() {
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
new file mode 100644
index 0000000..4144a54
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
@@ -0,0 +1,69 @@
+package org.apache.drill.exec.physical.impl.project;
+
+import static org.junit.Assert.assertEquals;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.record.vector.Fixed8;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.junit.After;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.yammer.metrics.MetricRegistry;
+
+public class TestSimpleProjection {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleProjection.class);
+ DrillConfig c = DrillConfig.create();
+
+
+ @Test
+ public void project(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Exception{
+ System.out.println(System.getProperty("java.class.path"));
+
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/project/test1.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+ while(exec.next()){
+ Fixed8 c1 = exec.getValueVectorById(new SchemaPath("col1", ExpressionPosition.UNKNOWN), Fixed8.class);
+ Fixed8 c2 = exec.getValueVectorById(new SchemaPath("col2", ExpressionPosition.UNKNOWN), Fixed8.class);
+ int x = 0;
+ for(int i =0; i < c1.getRecordCount(); i++){
+ assertEquals(c1.get(i)+1, c2.get(i));
+ x += c1.get(i);
+ }
+ System.out.println(x);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception{
+ // pause to get logger to catch up.
+ Thread.sleep(1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
index 99bf820..925faf7 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
@@ -66,7 +66,7 @@ public class ExpressionTreeMaterializerTest {
new NonStrictExpectations() {
{
- batch.getValueVector(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
+ batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
}
};
@@ -81,8 +81,8 @@ public class ExpressionTreeMaterializerTest {
public void testMaterializingLateboundTree(final @Injectable RecordBatch batch) throws SchemaChangeException {
new NonStrictExpectations() {
{
- batch.getValueVector(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BOOLEAN), -4);
- batch.getValueVector(new FieldReference("test1", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
+ batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BOOLEAN), -4);
+ batch.getValueVectorId(new FieldReference("test1", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
}
};
@@ -175,7 +175,7 @@ public class ExpressionTreeMaterializerTest {
new NonStrictExpectations() {
{
- batch.getValueVector(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
+ batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5);
}
};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java
new file mode 100644
index 0000000..4e5e4c7
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/test/generated/TestGeneratedClass.java
@@ -0,0 +1,15 @@
+package org.apache.drill.exec.test.generated;
+
+import org.apache.drill.exec.compile.JaninoClassCompiler;
+
+
+public class TestGeneratedClass {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestGeneratedClass.class);
+ public static void main(String[] args) throws Exception{
+ String s = "package org.apache.drill.exec.test.generated;\n\nimport org.apache.drill.exec.expr.holders.LongHolder;\nimport org.apache.drill.exec.ops.FragmentContext;\nimport org.apache.drill.exec.record.RecordBatch;\nimport org.apache.drill.exec.record.vector.NullableFixed8;\n\npublic class Test1 {\n\n NullableFixed8 vv0;\n NullableFixed8 vv5;\n NullableFixed8 vv7;\n NullableFixed8 vv12;\n\n public void setupEvaluators(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) {\n {\n {\n Object obj1 = (outgoing).getValueVectorById(1, NullableFixed8 .class);\n vv0 = ((NullableFixed8) obj1);\n Object obj6 = (outgoing).getValueVectorById(0, NullableFixed8 .class);\n vv5 = ((NullableFixed8) obj6);\n }\n {\n Object obj8 = (outgoing).getValueVectorById(1, NullableFixed8 .class);\n vv7 = ((NullableFixed8) obj8);\n Obj
ect obj13 = (outgoing).getValueVectorById(1, NullableFixed8 .class);\n vv12 = ((NullableFixed8) obj13);\n }\n }\n }\n\n public void doPerRecordWork(int inIndex, int outIndex) {\n {\n {\n LongHolder out2 = new LongHolder();\n out2 .value = vv0 .get((inIndex));\n LongHolder out3 = new LongHolder();\n out3 .value = 1L;\n LongHolder out4 = new LongHolder();\n {\n final LongHolder out = new LongHolder();\n final LongHolder left = out2;\n final LongHolder right = out3;\n out.value = left.value + right.value;\n\n out4 = out;\n }\n vv5 .set((outIndex), out4 .value);\n }\n {\n LongHolder out9 = new LongHolder();\n out9 .value = vv7 .get((inIndex));\n Long
Holder out10 = new LongHolder();\n out10 .value = 2L;\n LongHolder out11 = new LongHolder();\n {\n final LongHolder out = new LongHolder();\n final LongHolder left = out9;\n final LongHolder right = out10;\n out.value = left.value + right.value;\n\n out11 = out;\n }\n vv12 .set((outIndex), out11 .value);\n }\n }\n }\n\n}";
+ JaninoClassCompiler cc = new JaninoClassCompiler(ClassLoader.getSystemClassLoader());
+ byte[] b = cc.getClassByteCode("Test12", s);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
new file mode 100644
index 0000000..70a5115
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
@@ -0,0 +1,37 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"project",
+ exprs: [
+ { ref: "col1", expr:"red + 1" },
+ { ref: "col2", expr:"red + 2" }
+ ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/pom.xml b/sandbox/prototype/pom.xml
index a09d380..b7b9a40 100644
--- a/sandbox/prototype/pom.xml
+++ b/sandbox/prototype/pom.xml
@@ -105,6 +105,10 @@
</plugins>
<pluginManagement>
<plugins>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.15</version>
+ </plugin>
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
@@ -127,6 +131,19 @@
<ignore></ignore>
</action>
</pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>com.github.igor-petruk.protobuf</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <versionRange>[0.6.3,)</versionRange>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>