You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/07/20 03:58:07 UTC
[46/53] [abbrv] git commit: Working project operator. Update to ASM
4.1
Working project operator. Update to ASM 4.1
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/80b1d24f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/80b1d24f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/80b1d24f
Branch: refs/heads/master
Commit: 80b1d24f427f23feddd9c93ef0130c4657569f7e
Parents: ce0da88
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Jul 9 16:31:25 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Fri Jul 19 14:53:24 2013 -0700
----------------------------------------------------------------------
sandbox/prototype/common/pom.xml | 7 +
sandbox/prototype/exec/java-exec/pom.xml | 8 +-
.../drill/exec/compile/ClassTransformer.java | 224 ++++++++++++++-----
.../drill/exec/compile/JaninoClassCompiler.java | 7 +-
.../exec/compile/TemplateClassDefinition.java | 23 +-
.../apache/drill/exec/expr/CodeGenerator.java | 32 ++-
.../apache/drill/exec/expr/DrillBatchFunc.java | 10 +
.../drill/exec/expr/EvaluationVisitor.java | 76 ++++---
.../exec/expr/ExpressionTreeMaterializer.java | 2 +-
.../drill/exec/expr/fn/FunctionConverter.java | 9 +-
.../drill/exec/expr/fn/FunctionHolder.java | 32 ++-
.../drill/exec/expr/fn/impl/MathFunctions.java | 4 +-
.../drill/exec/expr/holders/BooleanHolder.java | 10 +
.../drill/exec/expr/holders/IntHolder.java | 10 +
.../drill/exec/expr/holders/LongHolder.java | 10 +
.../expr/holders/NullableBooleanHolder.java | 11 +
.../exec/expr/holders/NullableIntHolder.java | 11 +
.../exec/expr/holders/NullableLongHolder.java | 11 +
.../holders/ValueHolderImplmenetations.java | 44 ----
.../apache/drill/exec/ops/FragmentContext.java | 9 +-
.../exec/physical/impl/FilterRecordBatch.java | 4 +-
.../drill/exec/physical/impl/ImplCreator.java | 9 +-
.../drill/exec/physical/impl/ScanBatch.java | 2 +-
.../drill/exec/physical/impl/ScreenCreator.java | 7 +-
.../exec/physical/impl/WireRecordBatch.java | 2 +-
.../physical/impl/filter/ExampleFilter.java | 2 +-
.../physical/impl/filter/FilterTemplate.java | 48 ++++
.../SelectionVectorPopulationExpression.java | 39 ++++
.../impl/project/ProjectBatchCreator.java | 23 ++
.../physical/impl/project/ProjectEvaluator.java | 2 +-
.../impl/project/ProjectRecordBatch.java | 40 ++--
.../exec/physical/impl/project/Projector.java | 8 +-
.../impl/project/ProjectorTemplate.java | 38 +---
.../apache/drill/exec/record/RecordBatch.java | 4 +-
.../apache/drill/exec/record/SchemaBuilder.java | 3 +-
.../exec/record/vector/NullableFixed8.java | 7 +
.../exec/record/vector/SelectionVector2.java | 3 +
.../exec/compile/TestClassTransformation.java | 13 +-
.../apache/drill/exec/expr/ExpressionTest.java | 7 +-
.../exec/physical/impl/SimpleRootExec.java | 40 ++++
.../impl/project/TestSimpleProjection.java | 69 ++++++
.../record/ExpressionTreeMaterializerTest.java | 8 +-
.../exec/test/generated/TestGeneratedClass.java | 15 ++
.../src/test/resources/project/test1.json | 37 +++
sandbox/prototype/pom.xml | 17 ++
45 files changed, 750 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/common/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/pom.xml b/sandbox/prototype/common/pom.xml
index 4891191..a33ced9 100644
--- a/sandbox/prototype/common/pom.xml
+++ b/sandbox/prototype/common/pom.xml
@@ -74,6 +74,13 @@
<build>
<plugins>
<plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.15</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ </configuration>
+ </plugin>
+ <plugin>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index e348bc7..f2f0bff 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -13,14 +13,14 @@
<dependencies>
<dependency>
- <groupId>asm</groupId>
+ <groupId>org.ow2.asm</groupId>
<artifactId>asm-util</artifactId>
- <version>3.3.1</version>
+ <version>4.1</version>
</dependency>
<dependency>
- <groupId>asm</groupId>
+ <groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
- <version>3.3.1</version>
+ <version>4.1</version>
</dependency>
<dependency>
<groupId>com.sun.codemodel</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
index 079ab6c..4bf6e7e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
@@ -19,33 +19,37 @@ package org.apache.drill.exec.compile;
import java.io.File;
import java.io.IOException;
-import java.io.PrintWriter;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.util.Iterator;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.codehaus.commons.compiler.CompileException;
-import org.objectweb.asm.ClassAdapter;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.FieldVisitor;
import org.objectweb.asm.MethodVisitor;
-import org.objectweb.asm.commons.EmptyVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.Remapper;
import org.objectweb.asm.commons.RemappingClassAdapter;
import org.objectweb.asm.commons.RemappingMethodAdapter;
import org.objectweb.asm.commons.SimpleRemapper;
import org.objectweb.asm.tree.ClassNode;
import org.objectweb.asm.tree.FieldNode;
import org.objectweb.asm.tree.MethodNode;
-import org.objectweb.asm.util.TraceClassVisitor;
+import com.beust.jcommander.internal.Sets;
+import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.io.Files;
import com.google.common.io.Resources;
public class ClassTransformer {
@@ -53,76 +57,113 @@ public class ClassTransformer {
private AtomicLong index = new AtomicLong(0);
private AtomicLong interfaceIndex = new AtomicLong(0);
- private LoadingCache<String, byte[]> byteCode = CacheBuilder.newBuilder()
- .maximumSize(10000)
- .expireAfterWrite(10, TimeUnit.MINUTES)
- .build(new ClassBytesCacheLoader());
-
-
- private class ClassBytesCacheLoader extends CacheLoader<String, byte[]>{
+ private LoadingCache<String, byte[]> byteCode = CacheBuilder.newBuilder().maximumSize(10000)
+ .expireAfterWrite(10, TimeUnit.MINUTES).build(new ClassBytesCacheLoader());
+
+ private class ClassBytesCacheLoader extends CacheLoader<String, byte[]> {
public byte[] load(String path) throws ClassTransformationException, IOException {
URL u = this.getClass().getResource(path);
- if (u == null) throw new ClassTransformationException(String.format("Unable to find TemplateClass at path %s",path));
- return Resources.toByteArray(u);
+ if (u == null)
+ throw new ClassTransformationException(String.format("Unable to find TemplateClass at path %s", path));
+ return Resources.toByteArray(u);
}
};
-
+
private byte[] getClassByteCodeFromPath(String path) throws ClassTransformationException, IOException {
- try{
+ try {
return byteCode.get(path);
} catch (ExecutionException e) {
Throwable c = e.getCause();
- if(c instanceof ClassTransformationException) throw (ClassTransformationException) c;
- if(c instanceof IOException) throw (IOException) c;
+ if (c instanceof ClassTransformationException)
+ throw (ClassTransformationException) c;
+ if (c instanceof IOException)
+ throw (IOException) c;
throw new ClassTransformationException(c);
}
}
-
-
- @SuppressWarnings("unchecked")
- public <T, I> T getImplementationClass(QueryClassLoader classLoader,
- TemplateClassDefinition<T, I> templateDefinition, String internalClassBody, I initObject)
- throws ClassTransformationException {
- final String implClassName = templateDefinition.getTemplateClassName() + interfaceIndex.getAndIncrement();
+ public <T, I> T getImplementationClassByBody( //
+ QueryClassLoader classLoader, //
+ TemplateClassDefinition<T> templateDefinition, //
+ String internalClassBody //
+ ) throws ClassTransformationException {
final String materializedClassName = "org.apache.drill.generated."
- + templateDefinition.getExternalInterface().getSimpleName() + index.getAndIncrement();
- // final String materializedClassName = templateDefinition.getTemplateClassName();
+ + "Gen" + templateDefinition.getExternalInterface().getSimpleName() //
+ + index.getAndIncrement();
+ // Get Implementation Class
try {
-
- // Get Implementation Class
String classBody = ClassBodyBuilder.newBuilder() //
- .setClassName(implClassName) //
+ .setClassName(materializedClassName) //
.setImplementedInterfaces(templateDefinition.getInternalInterface()) //
.setBody(internalClassBody) //
.build();
- final byte[] implementationClass = classLoader.getClassByteCode(implClassName, classBody);
+ return getImplementationClass(classLoader, templateDefinition, classBody, materializedClassName);
+ } catch (IOException | CompileException e) {
+ throw new ClassTransformationException("Failure generating class body for runtime generated class.", e);
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T, I> T getImplementationClass( //
+ QueryClassLoader classLoader, //
+ TemplateClassDefinition<T> templateDefinition, //
+ String entireClass, //
+ String materializedClassName) throws ClassTransformationException {
+
+ try {
+
+ final byte[] implementationClass = classLoader.getClassByteCode(materializedClassName, entireClass);
// Get Template Class
final String templateClassName = templateDefinition.getTemplateClassName().replaceAll("\\.", File.separator);
final String templateClassPath = File.separator + templateClassName + ".class";
final byte[] templateClass = getClassByteCodeFromPath(templateClassPath);
-
+ int fileNum = new Random().nextInt(100);
+ Files.write(templateClass, new File(String.format("/tmp/%d-template.class", fileNum)));
// Generate Merge Class
- ClassNode impl = getClassNodeFromByteCode(implementationClass);
- // traceClassToSystemOut(implementationClass);
- // traceClassToSystemOut(templateClass);
- ClassWriter cw = new ClassWriter(0);
- MergeAdapter adapter = new MergeAdapter(cw, impl);
- ClassReader tReader = new ClassReader(templateClass);
- tReader.accept(adapter, 0);
-
- byte[] outputClass = cw.toByteArray();
-
- cw = new ClassWriter(0);
- RemappingClassAdapter r = new RemappingClassAdapter(cw, new SimpleRemapper(templateClassName,
- materializedClassName.replace('.', '/')));
- new ClassReader(outputClass).accept(r, 0);
- outputClass = cw.toByteArray();
- // traceClassToSystemOut(outputClass);
-
- // Load the class
- classLoader.injectByteCode(materializedClassName, outputClass);
+
+ // Setup adapters for merging, remapping class names and class writing. This is done in reverse order of how they
+ // will be evaluated.
+ String oldTemplateSlashName = templateDefinition.getTemplateClassName().replace('.', '/');
+ String materializedSlashName = materializedClassName.replace('.', '/');
+ RemapClasses remapper = new RemapClasses(oldTemplateSlashName, materializedSlashName);
+
+ {
+ ClassNode impl = getClassNodeFromByteCode(implementationClass);
+ ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
+
+ ClassVisitor remappingAdapter = new RemappingClassAdapter(cw, remapper);
+ MergeAdapter mergingAdapter = new MergeAdapter(oldTemplateSlashName, materializedSlashName, remappingAdapter,
+ impl);
+ ClassReader tReader = new ClassReader(templateClass);
+ tReader.accept(mergingAdapter, ClassReader.EXPAND_FRAMES);
+ byte[] outputClass = cw.toByteArray();
+ Files.write(outputClass, new File(String.format("/tmp/%d-output.class", fileNum)));
+ outputClass = cw.toByteArray();
+
+ // Load the class
+ classLoader.injectByteCode(materializedClassName, outputClass);
+ }
+
+ int i = 0;
+ for (String s : remapper.getSubclasses()) {
+ logger.debug("Setting up sub class {}", s);
+ // for each sub class, remap them into the new class.
+ String subclassPath = File.separator + s + ".class";
+ final byte[] bytecode = getClassByteCodeFromPath(subclassPath);
+ RemapClasses localRemapper = new RemapClasses(oldTemplateSlashName, materializedSlashName);
+ Preconditions.checkArgument(localRemapper.getSubclasses().isEmpty(), "Class transformations are only supported for classes that have a single level of inner classes.");
+ ClassWriter subcw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
+ ClassVisitor remap = new RemappingClassAdapter(subcw, localRemapper);
+ ClassReader reader = new ClassReader(bytecode);
+ reader.accept(remap, ClassReader.EXPAND_FRAMES);
+ byte[] newByteCode = subcw.toByteArray();
+ classLoader.injectByteCode(s.replace(oldTemplateSlashName, materializedSlashName).replace('/', '.'), newByteCode);
+ Files.write(subcw.toByteArray(), new File(String.format("/tmp/%d-sub-%d.class", fileNum, i)));
+ i++;
+ }
+
Class<?> c = classLoader.findClass(materializedClassName);
if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
return (T) c.newInstance();
@@ -131,7 +172,8 @@ public class ClassTransformer {
}
} catch (CompileException | IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
- throw new ClassTransformationException("Failure generating transformation classes.", e);
+ throw new ClassTransformationException(String.format(
+ "Failure generating transformation classes for value: \n %s", entireClass), e);
}
}
@@ -143,19 +185,37 @@ public class ClassTransformer {
return impl;
}
- private void traceClassToSystemOut(byte[] bytecode) {
- TraceClassVisitor tcv = new TraceClassVisitor(new EmptyVisitor(), new PrintWriter(System.out));
- ClassReader cr = new ClassReader(bytecode);
- cr.accept(tcv, 0);
- }
+ // private void traceClassToSystemOut(byte[] bytecode) {
+ // TraceClassVisitor tcv = new TraceClassVisitor(new EmptyVisitor(), new PrintWriter(System.out));
+ // ClassReader cr = new ClassReader(bytecode);
+ // cr.accept(tcv, 0);
+ // }
- public class MergeAdapter extends ClassAdapter {
+ public class MergeAdapter extends ClassVisitor {
private ClassNode classToMerge;
private String cname;
+ private String templateName;
+ private String newName;
- public MergeAdapter(ClassVisitor cv, ClassNode cn) {
- super(cv);
+ public MergeAdapter(String templateName, String newName, ClassVisitor cv, ClassNode cn) {
+ super(Opcodes.ASM4, cv);
this.classToMerge = cn;
+ this.templateName = templateName;
+ this.newName = newName.replace('.', '/');
+ ;
+
+ }
+
+ @Override
+ public void visitInnerClass(String name, String outerName, String innerName, int access) {
+ logger.debug(String.format(
+ "[Inner Class] Name: %s, outerName: %s, innerName: %s, templateName: %s, newName: %s.", name, outerName,
+ innerName, templateName, newName));
+ if (name.startsWith(templateName)) {
+ super.visitInnerClass(name.replace(templateName, newName), newName, innerName, access);
+ } else {
+ super.visitInnerClass(name, outerName, innerName, access);
+ }
}
// visit the class
@@ -193,7 +253,8 @@ public class ClassTransformer {
MethodNode mn = (MethodNode) it.next();
// skip the init.
- if (mn.name.equals("<init>")) continue;
+ if (mn.name.equals("<init>"))
+ continue;
String[] exceptions = new String[mn.exceptions.size()];
mn.exceptions.toArray(exceptions);
@@ -206,6 +267,47 @@ public class ClassTransformer {
}
super.visitEnd();
}
+
+ @Override
+ public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) {
+ return super.visitField(access, name, desc, signature, value);
+ }
+
+ }
+
+ static class RemapClasses extends Remapper {
+
+ final Set<String> subclasses = Sets.newHashSet();
+
+ String oldName;
+ String newName;
+
+ public RemapClasses(String oldName, String newName) {
+ super();
+ Preconditions.checkArgument(!newName.startsWith(oldName), "The new name of a class cannot start with the old name of a class, otherwise class renaming will cause problems.");
+ this.oldName = oldName;
+ this.newName = newName;
+ }
+
+ @Override
+ public String map(String typeName) {
+ // remap the names of all classes that start with the old class name.
+ if (typeName.startsWith(oldName)) {
+
+ // write down all the sub classes.
+ if (typeName.startsWith(oldName + "$")){
+ subclasses.add(typeName);
+ }
+
+ return typeName.replace(oldName, newName);
+ }
+ return typeName;
+ }
+
+ public Set<String> getSubclasses() {
+ return subclasses;
+ }
+
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
index 86fe58b..abe2afe 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
@@ -35,15 +35,16 @@ public class JaninoClassCompiler implements ClassCompiler{
private IClassLoader compilationClassLoader;
- private boolean debugLines;
- private boolean debugVars;
- private boolean debugSource;
+ private boolean debugLines = true;
+ private boolean debugVars = true;
+ private boolean debugSource = true;
public JaninoClassCompiler(ClassLoader parentClassLoader) {
this.compilationClassLoader = new ClassLoaderIClassLoader(parentClassLoader);
}
public byte[] getClassByteCode(final String className, final String code) throws CompileException, IOException, ClassNotFoundException, ClassTransformationException {
+ logger.debug("Compiling:\n {}", code);
StringReader reader = new StringReader(code);
Scanner scanner = new Scanner((String) null, reader);
Java.CompilationUnit compilationUnit = new Parser(scanner).parseCompilationUnit();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
index fee4c97..5a01dce 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
@@ -20,21 +20,25 @@ package org.apache.drill.exec.compile;
import java.lang.reflect.Method;
-public class TemplateClassDefinition<T, I>{
+public class TemplateClassDefinition<T>{
private final Class<T> externalInterface;
private final String templateClassName;
private final Class<?> internalInterface;
- private final Class<I> constructorObject;
+ private final String setupName;
+ private final String evalName;
- public TemplateClassDefinition(Class<T> externalInterface, String templateClassName, Class<?> internalInterface, Class<I> constructorObject) {
+
+ public TemplateClassDefinition(Class<T> externalInterface, String templateClassName, Class<?> internalInterface, String setupName, String evalName) {
super();
this.externalInterface = externalInterface;
- this.templateClassName = templateClassName;
+ this.templateClassName = templateClassName;
this.internalInterface = internalInterface;
- this.constructorObject = constructorObject;
+ this.setupName = setupName;
+ this.evalName = evalName;
}
+
public Class<T> getExternalInterface() {
return externalInterface;
}
@@ -48,8 +52,13 @@ public class TemplateClassDefinition<T, I>{
return templateClassName;
}
- public Class<I> getConstructorObject() {
- return constructorObject;
+ public String getSetupName() {
+ return setupName;
+ }
+
+
+ public String getEvalName() {
+ return evalName;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
index 77ae77d..ed6bd9b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
@@ -4,13 +4,14 @@ import java.io.IOException;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.compile.TemplateClassDefinition;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.BooleanHolder;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.IntHolder;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.LongHolder;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.NullableBooleanHolder;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.NullableIntHolder;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.NullableLongHolder;
+import org.apache.drill.exec.expr.holders.BooleanHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.LongHolder;
+import org.apache.drill.exec.expr.holders.NullableBooleanHolder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableLongHolder;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.RecordBatch;
@@ -26,7 +27,7 @@ import com.sun.codemodel.JMod;
import com.sun.codemodel.JType;
import com.sun.codemodel.JVar;
-public class CodeGenerator {
+public class CodeGenerator<T> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGenerator.class);
public JDefinedClass clazz;
@@ -37,17 +38,19 @@ public class CodeGenerator {
private final EvaluationVisitor evaluationVisitor;
private final String setupName;
private final String perRecordName;
-
+ private final TemplateClassDefinition<T> definition;
private JCodeModel model;
private int index = 0;
- public CodeGenerator(String setupName, String perRecordName, FunctionImplementationRegistry funcRegistry) {
+ public CodeGenerator(TemplateClassDefinition<T> definition, FunctionImplementationRegistry funcRegistry) {
super();
try{
- this.setupName = setupName;
- this.perRecordName = perRecordName;
+ this.definition = definition;
+ this.setupName = definition.getSetupName();
+ this.perRecordName = definition.getEvalName();
this.model = new JCodeModel();
this.clazz = model._package("org.apache.drill.exec.test.generated")._class("Test1");
+ clazz._implements(definition.getInternalInterface());
this.parentEvalBlock = new JBlock();
this.parentSetupBlock = new JBlock();
this.evaluationVisitor = new EvaluationVisitor(funcRegistry);
@@ -57,6 +60,7 @@ public class CodeGenerator {
}
public void addNextWrite(ValueVectorWriteExpression ex){
+ logger.debug("Adding next write {}", ex);
currentEvalBlock = new JBlock();
parentEvalBlock.add(currentEvalBlock);
currentSetupBlock = new JBlock();
@@ -68,6 +72,10 @@ public class CodeGenerator {
return currentEvalBlock;
}
+ public String getMaterializedClassName(){
+ return "org.apache.drill.exec.test.generated.Test1";
+ }
+
public JBlock getSetupBlock(){
return currentSetupBlock;
}
@@ -134,7 +142,7 @@ public class CodeGenerator {
}
- public class HoldingContainer{
+ public static class HoldingContainer{
private final JVar holder;
private final JFieldRef value;
private final JFieldRef isSet;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
new file mode 100644
index 0000000..5042708
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
@@ -0,0 +1,10 @@
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.exec.record.RecordBatch;
+
+public interface DrillBatchFunc {
+ public void setup(RecordBatch incoming);
+ public void eval();
+ public void batchReset();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index a5bc5fa..04f78d0 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -10,20 +10,24 @@ import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.FunctionHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.physical.impl.filter.SelectionVectorPopulationExpression;
+import org.apache.drill.exec.record.vector.SelectionVector2;
import org.apache.drill.exec.record.vector.TypeHelper;
+import com.google.common.base.Preconditions;
import com.sun.codemodel.JBlock;
import com.sun.codemodel.JClass;
import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JExpression;
import com.sun.codemodel.JType;
import com.sun.codemodel.JVar;
-public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, CodeGenerator, RuntimeException> {
+public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, CodeGenerator<?>, RuntimeException> {
private FunctionImplementationRegistry registry;
@@ -33,7 +37,7 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
}
@Override
- public HoldingContainer visitFunctionCall(FunctionCall call, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitFunctionCall(FunctionCall call, CodeGenerator<?> generator) throws RuntimeException {
HoldingContainer[] args = new HoldingContainer[call.args.size()];
for(int i = 0; i < call.args.size(); i++){
args[i] = call.args.get(i).accept(this, generator);
@@ -43,7 +47,7 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
}
@Override
- public HoldingContainer visitIfExpression(IfExpression ifExpr, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitIfExpression(IfExpression ifExpr, CodeGenerator<?> generator) throws RuntimeException {
JBlock local = generator.getBlock();
HoldingContainer output = generator.declare(ifExpr.getMajorType());
@@ -93,46 +97,46 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
}
@Override
- public HoldingContainer visitSchemaPath(SchemaPath path, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitSchemaPath(SchemaPath path, CodeGenerator<?> generator) throws RuntimeException {
throw new UnsupportedOperationException("All schema paths should have been replaced with ValueVectorExpressions.");
}
@Override
- public HoldingContainer visitLongConstant(LongExpression e, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitLongConstant(LongExpression e, CodeGenerator<?> generator) throws RuntimeException {
HoldingContainer out = generator.declare(e.getMajorType());
generator.getBlock().assign(out.getValue(), JExpr.lit(e.getLong()));
return out;
}
@Override
- public HoldingContainer visitDoubleConstant(DoubleExpression e, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitDoubleConstant(DoubleExpression e, CodeGenerator<?> generator) throws RuntimeException {
HoldingContainer out = generator.declare(e.getMajorType());
generator.getBlock().assign(out.getValue(), JExpr.lit(e.getDouble()));
return out;
}
@Override
- public HoldingContainer visitBooleanConstant(BooleanExpression e, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitBooleanConstant(BooleanExpression e, CodeGenerator<?> generator) throws RuntimeException {
HoldingContainer out = generator.declare(e.getMajorType());
generator.getBlock().assign(out.getValue(), JExpr.lit(e.getBoolean()));
return out;
}
-
-
@Override
- public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator generator) throws RuntimeException {
+ public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator<?> generator) throws RuntimeException {
if(e instanceof ValueVectorReadExpression){
return visitValueVectorExpression((ValueVectorReadExpression) e, generator);
}else if(e instanceof ValueVectorWriteExpression){
return visitValueVectorWriteExpression((ValueVectorWriteExpression) e, generator);
+ }else if(e instanceof SelectionVectorPopulationExpression){
+ return visitSelectionVectorExpression((SelectionVectorPopulationExpression) e, generator);
}else{
return super.visitUnknown(e, generator);
}
}
- private HoldingContainer visitValueVectorWriteExpression(ValueVectorWriteExpression e, CodeGenerator generator){
+ private HoldingContainer visitValueVectorWriteExpression(ValueVectorWriteExpression e, CodeGenerator<?> generator){
LogicalExpression child = e.getChild();
HoldingContainer hc = child.accept(this, generator);
JBlock block = generator.getBlock();
@@ -142,9 +146,11 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
JVar vv = generator.declareClassField("vv", vvType);
// get value vector in setup block.
- generator.getSetupBlock().assign(vv, JExpr.direct("outgoing").invoke("getValueVector") //
- .arg(JExpr.lit(e.getFieldId())) //
- .arg( ((JClass)vvType).dotclass()));
+ JVar obj = generator.getSetupBlock().decl( //
+ generator.getModel()._ref(Object.class), //
+ generator.getNextVar("obj"), //
+ JExpr.direct("outgoing").invoke("getValueVectorById").arg(JExpr.lit(e.getFieldId())).arg( ((JClass)vvType).dotclass()));
+ generator.getSetupBlock().assign(vv, JExpr.cast(vvType, obj));
if(hc.isOptional()){
vv.invoke("set").arg(JExpr.direct("outIndex"));
@@ -156,18 +162,19 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
return null;
}
- private HoldingContainer visitValueVectorExpression(ValueVectorReadExpression e, CodeGenerator generator) throws RuntimeException{
+ private HoldingContainer visitValueVectorExpression(ValueVectorReadExpression e, CodeGenerator<?> generator) throws RuntimeException{
// declare value vector
Class<?> vvClass = TypeHelper.getValueVectorClass(e.getMajorType().getMinorType(), e.getMajorType().getMode());
JType vvType = generator.getModel()._ref(vvClass);
JVar vv1 = generator.declareClassField("vv", vvType);
// get value vector from incoming batch and
- JInvocation getValue = JExpr //
- .invoke(JExpr.direct("incoming"), "getValueVector") //
- .arg(JExpr.lit(e.getFieldId())) //
- .arg( ((JClass)vvType).dotclass());
- generator.getSetupBlock().assign(vv1, getValue);
+ // get value vector in setup block.
+ JVar obj = generator.getSetupBlock().decl( //
+ generator.getModel()._ref(Object.class), //
+ generator.getNextVar("obj"), //
+ JExpr.direct("outgoing").invoke("getValueVectorById").arg(JExpr.lit(e.getFieldId())).arg( ((JClass)vvType).dotclass()));
+ generator.getSetupBlock().assign(vv1, JExpr.cast(vvType, obj));
// evaluation work.
HoldingContainer out = generator.declare(e.getMajorType());
@@ -175,25 +182,42 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
if(out.isOptional()){
JBlock blk = generator.getBlock();
- blk.assign(out.getIsSet(), vv1.invoke("isSet").arg(JExpr.direct("index")));
+ blk.assign(out.getIsSet(), vv1.invoke("isSet").arg(JExpr.direct("inIndex")));
JConditional jc = blk._if(out.getIsSet());
jc._then() //
- .assign(out.getValue(), vv1.invoke("get").arg(JExpr.direct("index"))); //
+ .assign(out.getValue(), vv1.invoke("get").arg(JExpr.direct("inIndex"))); //
//.assign(out.getIsSet(), JExpr.lit(1));
//jc._else()
//.assign(out.getIsSet(), JExpr.lit(0));
}else{
- generator.getBlock().assign(out.getValue(), vv1.invoke("get").arg(JExpr.direct("index")));
+ generator.getBlock().assign(out.getValue(), vv1.invoke("get").arg(JExpr.direct("inIndex")));
}
return out;
}
+ private HoldingContainer visitSelectionVectorExpression(SelectionVectorPopulationExpression e, CodeGenerator<?> generator){
+ JType svClass = generator.getModel()._ref(SelectionVector2.class);
+ JVar sv = generator.declareClassField("sv", svClass);
+ JVar index = generator.declareClassField("svIndex", generator.getModel().CHAR);
+ LogicalExpression child = e.getChild();
+ Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN));
+ HoldingContainer hc = child.accept(this, generator);
+ generator.getBlock()._return(hc.getValue());
+
+// JBlock blk = generator.getSetupBlock();
+// blk.assign(sv, JExpr.direct("outgoing").invoke("getSelectionVector2"));
+// JConditional jc = blk._if(hc.getValue());
+// JBlock body = jc._then();
+// body.add(sv.invoke("set").arg(index).arg(JExpr.direct("inIndex")));
+// body.assign(index, index.plus(JExpr.lit(1)));
+ return null;
+ }
@Override
- public HoldingContainer visitQuotedStringConstant(QuotedString e, CodeGenerator CodeGenerator) throws RuntimeException {
- throw new UnsupportedOperationException("We don't yet support string literals as we need to use the valuevector classes and internal vectors.");
+ public HoldingContainer visitQuotedStringConstant(QuotedString e, CodeGenerator<?> CodeGenerator) throws RuntimeException {
+ throw new UnsupportedOperationException("We don't yet support string literals as we need to build the string value holders.");
// JExpr stringLiteral = JExpr.lit(e.value);
// CodeGenerator.block.decl(stringLiteral.invoke("getBytes").arg(JExpr.ref(Charsets.UTF_8));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index bc44490..72e5c93 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -97,7 +97,7 @@ public class ExpressionTreeMaterializer {
@Override
public LogicalExpression visitSchemaPath(SchemaPath path) {
logger.debug("Visiting schema path {}", path);
- TypedFieldId tfId = batch.getValueVector(path);
+ TypedFieldId tfId = batch.getValueVectorId(path);
if (tfId == null) {
return NullExpression.INSTANCE;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 7276e7d..84f04f0 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -16,6 +16,7 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.WorkSpace;
import org.apache.drill.exec.expr.fn.FunctionHolder.ValueReference;
+import org.apache.drill.exec.expr.fn.FunctionHolder.WorkspaceReference;
import org.apache.drill.exec.expr.holders.ValueHolder;
import org.codehaus.commons.compiler.CompileException;
import org.codehaus.janino.Java;
@@ -43,7 +44,8 @@ public class FunctionConverter {
// start by getting field information.
List<ValueReference> params = Lists.newArrayList();
- List<String> workspaceFields = Lists.newArrayList();
+ List<WorkspaceReference> workspaceFields = Lists.newArrayList();
+
ValueReference outputField = null;
@@ -96,7 +98,7 @@ public class FunctionConverter {
}else{
// workspace work.
- workspaceFields.add(field.getName());
+ workspaceFields.add(new WorkspaceReference(field.getType(), field.getName()));
}
}
@@ -118,7 +120,8 @@ public class FunctionConverter {
// return holder
ValueReference[] ps = params.toArray(new ValueReference[params.size()]);
- FunctionHolder fh = new FunctionHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, methods);
+ WorkspaceReference[] works = workspaceFields.toArray(new WorkspaceReference[workspaceFields.size()]);
+ FunctionHolder fh = new FunctionHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, works, methods);
return fh;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
index 460169f..f7ecd02 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
@@ -1,5 +1,6 @@
package org.apache.drill.exec.expr.fn;
+import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.Map;
@@ -32,13 +33,15 @@ public class FunctionHolder {
private String evalBody;
private String addBody;
private String setupBody;
+ private WorkspaceReference[] workspaceVars;
private ValueReference[] parameters;
private ValueReference returnValue;
- public FunctionHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, String functionName, ValueReference[] parameters, ValueReference returnValue, Map<String, String> methods) {
+ public FunctionHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map<String, String> methods) {
super();
this.scope = scope;
this.nullHandling = nullHandling;
+ this.workspaceVars = workspaceVars;
this.isBinaryCommutative = isBinaryCommutative;
this.functionName = functionName;
this.setupBody = methods.get("setup");
@@ -48,9 +51,9 @@ public class FunctionHolder {
this.returnValue = returnValue;
}
- public HoldingContainer generateEvalBody(CodeGenerator g, HoldingContainer[] inputVariables){
+ public HoldingContainer generateEvalBody(CodeGenerator<?> g, HoldingContainer[] inputVariables){
- g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName));
+ //g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName));
JBlock sub = new JBlock(true, true);
@@ -87,6 +90,16 @@ public class FunctionHolder {
// add the subblock after the out declaration.
g.getBlock().add(sub);
+ JVar[] workspaceJVars = new JVar[workspaceVars.length];
+ for(int i =0 ; i < workspaceVars.length; i++){
+ workspaceJVars[i] = g.declareClassField("work", g.getModel()._ref(workspaceVars[i].type)));
+ }
+
+ for(WorkspaceReference r : workspaceVars){
+ g.declareClassField(, t)
+ }
+
+ g.declareClassField(prefix, t)
// locally name external blocks.
// internal out value.
@@ -99,12 +112,13 @@ public class FunctionHolder {
sub.decl(JMod.FINAL, inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
}
+
// add function body.
sub.directStatement(evalBody);
sub.assign(out.getHolder(), internalOutput);
- g.getBlock().directStatement(String.format("//---- end of eval portion of %s function. ----//\n", functionName));
+ //g.getBlock().directStatement(String.format("//---- end of eval portion of %s function. ----//\n", functionName));
return out;
}
@@ -144,6 +158,16 @@ public class FunctionHolder {
}
+ public static class WorkspaceReference{
+ Class<?> type;
+ String name;
+ public WorkspaceReference(Class<?> type, String name) {
+ super();
+ this.type = type;
+ this.name = name;
+ }
+
+ }
@Override
public String toString() {
final int maxLen = 10;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index 702a0ad..939e997 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -6,8 +6,8 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.IntHolder;
-import org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.LongHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.LongHolder;
import org.apache.drill.exec.record.RecordBatch;
public class MathFunctions{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
new file mode 100644
index 0000000..70bcd10
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
@@ -0,0 +1,10 @@
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public final class BooleanHolder implements ValueHolder{
+ public static final MajorType TYPE = Types.required(MinorType.BOOLEAN);
+ public int value;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
new file mode 100644
index 0000000..81c82b9
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
@@ -0,0 +1,10 @@
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public final class IntHolder implements ValueHolder{
+ public static final MajorType TYPE = Types.required(MinorType.INT);
+ public int value;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
new file mode 100644
index 0000000..c73f9ad
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
@@ -0,0 +1,10 @@
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public final class LongHolder implements ValueHolder {
+ public static final MajorType TYPE = Types.required(MinorType.BIGINT);
+ public long value;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
new file mode 100644
index 0000000..f6d2f2e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
@@ -0,0 +1,11 @@
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public final class NullableBooleanHolder implements ValueHolder {
+ public static final MajorType TYPE = Types.optional(MinorType.BOOLEAN);
+ public int value;
+ public int isSet;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
new file mode 100644
index 0000000..7eca21e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
@@ -0,0 +1,11 @@
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public final class NullableIntHolder implements ValueHolder {
+ public static final MajorType TYPE = Types.optional(MinorType.INT);
+ public int value;
+ public int isSet;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
new file mode 100644
index 0000000..d1be8f1
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
@@ -0,0 +1,11 @@
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public final class NullableLongHolder implements ValueHolder {
+ public static final MajorType TYPE = Types.optional(MinorType.BIGINT);
+ public long value;
+ public int isSet;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
deleted file mode 100644
index adf97fe..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package org.apache.drill.exec.expr.holders;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-
-public class ValueHolderImplmenetations {
-
- private ValueHolderImplmenetations(){}
-
- public final static class BooleanHolder implements ValueHolder{
- public static final MajorType TYPE = Types.required(MinorType.BOOLEAN);
- public int value;
- }
-
- public final static class NullableBooleanHolder implements ValueHolder {
- public static final MajorType TYPE = Types.optional(MinorType.BOOLEAN);
- public int value;
- public int isSet;
- }
-
- public final static class IntHolder implements ValueHolder{
- public static final MajorType TYPE = Types.required(MinorType.INT);
- public int value;
- }
-
- public final static class NullableIntHolder implements ValueHolder {
- public static final MajorType TYPE = Types.optional(MinorType.INT);
- public int value;
- public int isSet;
- }
-
- public final static class LongHolder implements ValueHolder {
- public static final MajorType TYPE = Types.required(MinorType.BIGINT);
- public long value;
- }
-
- public final static class NullableLongHolder implements ValueHolder {
- public static final MajorType TYPE = Types.optional(MinorType.BIGINT);
- public long value;
- public int isSet;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 1dd7de8..876b873 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -103,14 +103,11 @@ public class FragmentContext {
return context.getAllocator();
}
- public <T> T getImplementationClass(TemplateClassDefinition<T, Void> templateDefinition, CodeGenerator cg) throws ClassTransformationException, IOException{
- return transformer.getImplementationClass(this.loader, templateDefinition, cg.generate(), null);
- }
-
- public FilteringRecordBatchTransformer getFilteringExpression(LogicalExpression expr){
- return null;
+ public <T> T getImplementationClass(TemplateClassDefinition<T> templateDefinition, CodeGenerator<T> cg) throws ClassTransformationException, IOException{
+ return transformer.getImplementationClass(this.loader, templateDefinition, cg.generate(), cg.getMaterializedClassName());
}
+
public void addMetricsToStatus(FragmentStatus.Builder stats){
stats.setBatchesCompleted(batchesCompleted.get());
stats.setDataProcessed(dataProcessed.get());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
index 80d48f4..1ddc3f9 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
@@ -78,7 +78,7 @@ public abstract class FilterRecordBatch implements RecordBatch {
}
@Override
- public TypedFieldId getValueVector(SchemaPath path) {
+ public TypedFieldId getValueVectorId(SchemaPath path) {
return null;
}
@@ -111,7 +111,7 @@ public abstract class FilterRecordBatch implements RecordBatch {
IterOutcome o = incoming.next();
switch (o) {
case OK_NEW_SCHEMA:
- transformer = incoming.getContext().getFilteringExpression(null);
+ transformer = null;
schema = transformer.getSchema();
// fall through to ok.
case OK:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
index d98c107..739c0d4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
@@ -28,9 +28,11 @@ import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Scan;
import org.apache.drill.exec.physical.config.MockScanBatchCreator;
import org.apache.drill.exec.physical.config.MockScanPOP;
+import org.apache.drill.exec.physical.config.Project;
import org.apache.drill.exec.physical.config.RandomReceiver;
import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.config.SingleSender;
+import org.apache.drill.exec.physical.impl.project.ProjectBatchCreator;
import org.apache.drill.exec.record.RecordBatch;
import com.google.common.base.Preconditions;
@@ -43,6 +45,7 @@ public class ImplCreator extends AbstractPhysicalVisitor<RecordBatch, FragmentCo
private ScreenCreator sc = new ScreenCreator();
private RandomReceiverCreator rrc = new RandomReceiverCreator();
private SingleSenderCreator ssc = new SingleSenderCreator();
+ private ProjectBatchCreator pbc = new ProjectBatchCreator();
private RootExec root = null;
private ImplCreator(){}
@@ -51,7 +54,11 @@ public class ImplCreator extends AbstractPhysicalVisitor<RecordBatch, FragmentCo
return root;
}
-
+ @Override
+ public RecordBatch visitProject(Project op, FragmentContext context) throws ExecutionSetupException {
+ return pbc.getBatch(context, op, getChildren(op, context));
+ }
+
@Override
public RecordBatch visitScan(Scan<?> scan, FragmentContext context) throws ExecutionSetupException {
Preconditions.checkNotNull(scan);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 1e0c000..be80349 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -173,7 +173,7 @@ public class ScanBatch implements RecordBatch {
}
@Override
- public TypedFieldId getValueVector(SchemaPath path) {
+ public TypedFieldId getValueVectorId(SchemaPath path) {
return holder.getValueVector(path);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
index 3819036..516b5af 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
@@ -50,7 +50,7 @@ public class ScreenCreator implements RootCreator<Screen>{
}
- private static class ScreenRoot implements RootExec{
+ static class ScreenRoot implements RootExec{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScreenRoot.class);
volatile boolean ok = true;
@@ -138,6 +138,11 @@ public class ScreenCreator implements RootCreator<Screen>{
}
}
+
+ RecordBatch getIncoming() {
+ return incoming;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index f4921b5..7b2a533 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -85,7 +85,7 @@ public class WireRecordBatch implements RecordBatch{
}
@Override
- public TypedFieldId getValueVector(SchemaPath path) {
+ public TypedFieldId getValueVectorId(SchemaPath path) {
return batchLoader.getValueVector(path);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
index 158350f..c4a4032 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
@@ -78,7 +78,7 @@ public class ExampleFilter implements RecordBatch {
}
@Override
- public TypedFieldId getValueVector(SchemaPath path) {
+ public TypedFieldId getValueVectorId(SchemaPath path) {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
new file mode 100644
index 0000000..d2983ad
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
@@ -0,0 +1,48 @@
+package org.apache.drill.exec.physical.impl.filter;
+
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.vector.SelectionVector2;
+
+public abstract class FilterTemplate {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FilterTemplate.class);
+
+ SelectionVector2 outgoingSelectionVector;
+ SelectionVector2 incomingSelectionVector;
+
+ public void setup(RecordBatch incoming, RecordBatch outgoing){
+ outgoingSelectionVector = outgoing.getSelectionVector2();
+
+ switch(incoming.getSchema().getSelectionVector()){
+ case NONE:
+ break;
+ case TWO_BYTE:
+ this.incomingSelectionVector = incoming.getSelectionVector2();
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public void filterBatchSV2(int recordCount){
+ int svIndex = 0;
+ for(char i =0; i < recordCount; i++){
+ if(include(i)){
+ outgoingSelectionVector.setIndex(svIndex, i);
+ svIndex+=2;
+ }
+ }
+ }
+
+ public void filterBatchNoSV(int recordCount){
+ int svIndex = 0;
+ for(char i =0; i < recordCount; i++){
+
+ if(include(i)){
+ outgoingSelectionVector.setIndex(svIndex, i);
+ svIndex+=2;
+ }
+ }
+ }
+
+ protected abstract boolean include(int index);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
new file mode 100644
index 0000000..f253695
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
@@ -0,0 +1,39 @@
+package org.apache.drill.exec.physical.impl.filter;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+public class SelectionVectorPopulationExpression implements LogicalExpression{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVectorPopulationExpression.class);
+
+ private LogicalExpression child;
+
+ public SelectionVectorPopulationExpression(LogicalExpression child) {
+ this.child = child;
+ }
+
+ public LogicalExpression getChild() {
+ return child;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return Types.NULL;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitUnknown(this, value);
+ }
+
+ @Override
+ public ExpressionPosition getPosition() {
+ return ExpressionPosition.UNKNOWN;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
new file mode 100644
index 0000000..3538469
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
@@ -0,0 +1,23 @@
+package org.apache.drill.exec.physical.impl.project;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.Project;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+
+import com.google.common.base.Preconditions;
+
+public class ProjectBatchCreator implements BatchCreator<Project>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, Project config, List<RecordBatch> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.size() == 1);
+ return new ProjectRecordBatch(config, children.iterator().next(), context);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
index 69daae0..86caf28 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
@@ -7,6 +7,6 @@ import org.apache.drill.exec.record.RecordBatch;
public interface ProjectEvaluator {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectEvaluator.class);
- public abstract void setupEvaluators(FragmentContext context, RecordBatch incoming) throws SchemaChangeException;
+ public abstract void setupEvaluators(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException;
public abstract void doPerRecordWork(int inIndex, int outIndex);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index cfdb7bc..7a3fec1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -5,6 +5,7 @@ import java.util.List;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
@@ -18,6 +19,7 @@ import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Project;
+import org.apache.drill.exec.physical.impl.VectorHolder;
import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
import org.apache.drill.exec.proto.SchemaDefProtos.NamePart;
import org.apache.drill.exec.proto.SchemaDefProtos.NamePart.Type;
@@ -32,6 +34,7 @@ import org.apache.drill.exec.record.vector.SelectionVector4;
import org.apache.drill.exec.record.vector.TypeHelper;
import org.apache.drill.exec.record.vector.ValueVector;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -45,6 +48,7 @@ public class ProjectRecordBatch implements RecordBatch{
private Projector projector;
private List<ValueVector<?>> allocationVectors;
private List<ValueVector<?>> outputVectors;
+ private VectorHolder vh;
public ProjectRecordBatch(Project pop, RecordBatch incoming, FragmentContext context){
@@ -86,19 +90,20 @@ public class ProjectRecordBatch implements RecordBatch{
}
@Override
- public TypedFieldId getValueVector(SchemaPath path) {
- return null;
+ public TypedFieldId getValueVectorId(SchemaPath path) {
+ return vh.getValueVector(path);
}
@Override
- public <T extends ValueVector<T>> T getValueVectorById(int fieldId, Class<?> vvClass) {
- return null;
+ public <T extends ValueVector<T>> T getValueVectorById(int fieldId, Class<?> clazz) {
+ return vh.getValueVector(fieldId, clazz);
}
@Override
public IterOutcome next() {
IterOutcome upstream = incoming.next();
+ logger.debug("Upstream... {}", upstream);
switch(upstream){
case NONE:
case NOT_YET:
@@ -109,6 +114,7 @@ public class ProjectRecordBatch implements RecordBatch{
projector = createNewProjector();
}catch(SchemaChangeException ex){
incoming.kill();
+ logger.error("Failure during query", ex);
context.fail(ex);
return IterOutcome.STOP;
}
@@ -119,6 +125,9 @@ public class ProjectRecordBatch implements RecordBatch{
v.allocateNew(recordCount);
}
projector.projectRecords(recordCount, 0);
+ for(ValueVector<?> v : this.outputVectors){
+ v.setRecordCount(recordCount);
+ }
return upstream; // change if upstream changed, otherwise normal.
default:
throw new UnsupportedOperationException();
@@ -134,24 +143,21 @@ public class ProjectRecordBatch implements RecordBatch{
}
}
this.outputVectors = Lists.newArrayList();
-
+ this.vh = new VectorHolder(outputVectors);
final List<NamedExpression> exprs = pop.getExprs();
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPairing<?>> transfers = Lists.newArrayList();
- final CodeGenerator cg = new CodeGenerator("setupEvaluators", "doPerRecordWork", context.getFunctionRegistry());
+ final CodeGenerator<Projector> cg = new CodeGenerator<Projector>(Projector.TEMPLATE_DEFINITION, context.getFunctionRegistry());
for(int i =0; i < exprs.size(); i++){
final NamedExpression namedExpression = exprs.get(i);
- final MaterializedField outputField = getMaterializedField(namedExpression);
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector);
+ final MaterializedField outputField = getMaterializedField(namedExpression.getRef(), expr);
if(collector.hasErrors()){
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
}
-
-
-
// add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
if(expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVector() == SelectionVectorMode.NONE){
ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
@@ -179,21 +185,21 @@ public class ProjectRecordBatch implements RecordBatch{
this.outSchema = bldr.build();
try {
- return context.getImplementationClass(Projector.TEMPLATE_DEFINITION, cg);
+ Projector projector = context.getImplementationClass(Projector.TEMPLATE_DEFINITION, cg);
+ projector.setup(context, incoming, this, transfers);
+ return projector;
} catch (ClassTransformationException | IOException e) {
throw new SchemaChangeException("Failure while attempting to load generated class", e);
}
}
-
@Override
public WritableBatch getWritableBatch() {
- return null;
+ return WritableBatch.get(incoming.getRecordCount(), outputVectors);
}
-
- private MaterializedField getMaterializedField(NamedExpression ex){
- return new MaterializedField(getFieldDef(ex.getRef(), ex.getExpr().getMajorType()));
+ private MaterializedField getMaterializedField(FieldReference reference, LogicalExpression expr){
+ return new MaterializedField(getFieldDef(reference, expr.getMajorType()));
}
private FieldDef getFieldDef(SchemaPath path, MajorType type){
@@ -212,7 +218,9 @@ public class ProjectRecordBatch implements RecordBatch{
}else{
parts.add(NamePart.newBuilder().setType(Type.NAME).setName(seg.getNameSegment().getPath().toString()).build());
}
+ seg = seg.getChild();
}
return parts;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
index 31c418c..8f22b40 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
@@ -9,12 +9,12 @@ import org.apache.drill.exec.record.RecordBatch;
public interface Projector {
- public abstract void setup(FragmentContext context, RecordBatch incoming, List<TransferPairing<?>> transfers) throws SchemaChangeException;
+ public abstract void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, List<TransferPairing<?>> transfers) throws SchemaChangeException;
- public abstract void projectRecords(int recordCount, int firstOutputIndex);
+ public abstract int projectRecords(int recordCount, int firstOutputIndex);
- public static TemplateClassDefinition<Projector, Void> TEMPLATE_DEFINITION = new TemplateClassDefinition<Projector, Void>( //
- Projector.class, "org.apache.drill.exec.physical.impl.project.ProjectTemplate", ProjectEvaluator.class, Void.class);
+ public static TemplateClassDefinition<Projector> TEMPLATE_DEFINITION = new TemplateClassDefinition<Projector>( //
+ Projector.class, "org.apache.drill.exec.physical.impl.project.ProjectorTemplate", ProjectEvaluator.class, "setupEvaluators", "doPerRecordWork");
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
index 60af7d2..1163fef 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
@@ -2,50 +2,28 @@ package org.apache.drill.exec.physical.impl.project;
import java.util.List;
-import org.apache.drill.common.expression.ErrorCollector;
-import org.apache.drill.common.expression.ErrorCollectorImpl;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.ValueVectorReadExpression;
-import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.config.Project;
-import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
-import org.apache.drill.exec.proto.SchemaDefProtos.NamePart;
-import org.apache.drill.exec.proto.SchemaDefProtos.NamePart.Type;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.vector.SelectionVector2;
import org.apache.drill.exec.record.vector.SelectionVector4;
-import org.apache.drill.exec.record.vector.TypeHelper;
-import org.apache.drill.exec.record.vector.ValueVector;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
public abstract class ProjectorTemplate implements Projector {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectorTemplate.class);
-
+
private ImmutableList<TransferPairing<?>> transfers;
private SelectionVector2 vector2;
private SelectionVector4 vector4;
private SelectionVectorMode svMode;
- public ProjectorTemplate(final FragmentContext context, final RecordBatch incomingBatch, final Project pop, FunctionImplementationRegistry funcRegistry) throws SchemaChangeException{
- super();
+ public ProjectorTemplate() throws SchemaChangeException{
}
@Override
- public final void projectRecords(final int recordCount, int firstOutputIndex) {
+ public final int projectRecords(final int recordCount, int firstOutputIndex) {
switch(svMode){
case FOUR_BYTE:
throw new UnsupportedOperationException();
@@ -56,7 +34,7 @@ public abstract class ProjectorTemplate implements Projector {
for(int i = 0; i < count; i+=2, firstOutputIndex++){
doPerRecordWork(vector2.getIndex(i), firstOutputIndex);
}
- return;
+ return recordCount;
case NONE:
@@ -68,7 +46,7 @@ public abstract class ProjectorTemplate implements Projector {
for (int i = 0; i < countN; i++, firstOutputIndex++) {
doPerRecordWork(i, firstOutputIndex);
}
- return;
+ return recordCount;
default:
@@ -77,7 +55,7 @@ public abstract class ProjectorTemplate implements Projector {
}
@Override
- public final void setup(FragmentContext context, RecordBatch incoming, List<TransferPairing<?>> transfers) throws SchemaChangeException{
+ public final void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, List<TransferPairing<?>> transfers) throws SchemaChangeException{
this.svMode = incoming.getSchema().getSelectionVector();
switch(svMode){
@@ -89,10 +67,10 @@ public abstract class ProjectorTemplate implements Projector {
break;
}
this.transfers = ImmutableList.copyOf(transfers);
- setupEvaluators(context, incoming);
+ setupEvaluators(context, incoming, outgoing);
}
- protected abstract void setupEvaluators(FragmentContext context, RecordBatch incoming) throws SchemaChangeException;
+ protected abstract void setupEvaluators(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException;
protected abstract void doPerRecordWork(int inIndex, int outIndex);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
index 40447ec..0edb93e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
@@ -85,10 +85,10 @@ public interface RecordBatch {
* @param path The path where the vector should be located.
* @return The local field id associated with this vector.
*/
- public abstract TypedFieldId getValueVector(SchemaPath path);
+ public abstract TypedFieldId getValueVectorId(SchemaPath path);
- public abstract <T extends ValueVector<T>> T getValueVectorById(int fieldId, Class<?> vvClass);
+ public abstract <T extends ValueVector<T>> T getValueVectorById(int fieldId, Class<?> clazz);
/**
* Update the data in each Field reading interface for the next range of records. Once a RecordBatch returns an
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
index 20cc82c..0989c1d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
@@ -21,6 +21,7 @@ import java.util.List;
import java.util.Set;
import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import com.beust.jcommander.internal.Sets;
import com.google.common.collect.Lists;
@@ -32,7 +33,7 @@ import com.google.common.collect.Lists;
public class SchemaBuilder {
private Set<MaterializedField> fields = Sets.newHashSet();
- private BatchSchema.SelectionVectorMode selectionVectorMode;
+ private BatchSchema.SelectionVectorMode selectionVectorMode = SelectionVectorMode.NONE;
SchemaBuilder() {
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
index 3fc39eb..48aa0c2 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
@@ -32,5 +32,12 @@ public final class NullableFixed8 extends NullableValueVector<NullableFixed8, Fi
return new Fixed8(this.field, allocator);
}
+ public long get(int index){
+ return 1l;
+ }
+
+ public void set(int index, long value){
+
+ }
}
Re: [46/53] [abbrv] git commit: Working project operator. Update to
ASM 4.1
Posted by Ted Dunning <te...@gmail.com>.
On Fri, Jul 19, 2013 at 6:58 PM, <ja...@apache.org> wrote:
> Working project operator. Update to ASM 4.1
>
>
> Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
> Commit:
> http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/80b1d24f
> Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/80b1d24f
> Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/80b1d24f
>
> Branch: refs/heads/master
> Commit: 80b1d24f427f23feddd9c93ef0130c4657569f7e
> Parents: ce0da88
> Author: Jacques Nadeau <ja...@apache.org>
> Authored: Tue Jul 9 16:31:25 2013 -0700
> Committer: Jacques Nadeau <ja...@apache.org>
> Committed: Fri Jul 19 14:53:24 2013 -0700
>
> ----------------------------------------------------------------------
> sandbox/prototype/common/pom.xml | 7 +
> sandbox/prototype/exec/java-exec/pom.xml | 8 +-
> .../drill/exec/compile/ClassTransformer.java | 224 ++++++++++++++-----
> .../drill/exec/compile/JaninoClassCompiler.java | 7 +-
> .../exec/compile/TemplateClassDefinition.java | 23 +-
> .../apache/drill/exec/expr/CodeGenerator.java | 32 ++-
> .../apache/drill/exec/expr/DrillBatchFunc.java | 10 +
> .../drill/exec/expr/EvaluationVisitor.java | 76 ++++---
> .../exec/expr/ExpressionTreeMaterializer.java | 2 +-
> .../drill/exec/expr/fn/FunctionConverter.java | 9 +-
> .../drill/exec/expr/fn/FunctionHolder.java | 32 ++-
> .../drill/exec/expr/fn/impl/MathFunctions.java | 4 +-
> .../drill/exec/expr/holders/BooleanHolder.java | 10 +
> .../drill/exec/expr/holders/IntHolder.java | 10 +
> .../drill/exec/expr/holders/LongHolder.java | 10 +
> .../expr/holders/NullableBooleanHolder.java | 11 +
> .../exec/expr/holders/NullableIntHolder.java | 11 +
> .../exec/expr/holders/NullableLongHolder.java | 11 +
> .../holders/ValueHolderImplmenetations.java | 44 ----
> .../apache/drill/exec/ops/FragmentContext.java | 9 +-
> .../exec/physical/impl/FilterRecordBatch.java | 4 +-
> .../drill/exec/physical/impl/ImplCreator.java | 9 +-
> .../drill/exec/physical/impl/ScanBatch.java | 2 +-
> .../drill/exec/physical/impl/ScreenCreator.java | 7 +-
> .../exec/physical/impl/WireRecordBatch.java | 2 +-
> .../physical/impl/filter/ExampleFilter.java | 2 +-
> .../physical/impl/filter/FilterTemplate.java | 48 ++++
> .../SelectionVectorPopulationExpression.java | 39 ++++
> .../impl/project/ProjectBatchCreator.java | 23 ++
> .../physical/impl/project/ProjectEvaluator.java | 2 +-
> .../impl/project/ProjectRecordBatch.java | 40 ++--
> .../exec/physical/impl/project/Projector.java | 8 +-
> .../impl/project/ProjectorTemplate.java | 38 +---
> .../apache/drill/exec/record/RecordBatch.java | 4 +-
> .../apache/drill/exec/record/SchemaBuilder.java | 3 +-
> .../exec/record/vector/NullableFixed8.java | 7 +
> .../exec/record/vector/SelectionVector2.java | 3 +
> .../exec/compile/TestClassTransformation.java | 13 +-
> .../apache/drill/exec/expr/ExpressionTest.java | 7 +-
> .../exec/physical/impl/SimpleRootExec.java | 40 ++++
> .../impl/project/TestSimpleProjection.java | 69 ++++++
> .../record/ExpressionTreeMaterializerTest.java | 8 +-
> .../exec/test/generated/TestGeneratedClass.java | 15 ++
> .../src/test/resources/project/test1.json | 37 +++
> sandbox/prototype/pom.xml | 17 ++
> 45 files changed, 750 insertions(+), 247 deletions(-)
> ----------------------------------------------------------------------
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/common/pom.xml
> ----------------------------------------------------------------------
> diff --git a/sandbox/prototype/common/pom.xml
> b/sandbox/prototype/common/pom.xml
> index 4891191..a33ced9 100644
> --- a/sandbox/prototype/common/pom.xml
> +++ b/sandbox/prototype/common/pom.xml
> @@ -74,6 +74,13 @@
> <build>
> <plugins>
> <plugin>
> + <artifactId>maven-surefire-plugin</artifactId>
> + <version>2.15</version>
> + <configuration>
> + <useSystemClassLoader>false</useSystemClassLoader>
> + </configuration>
> + </plugin>
> + <plugin>
> <artifactId>maven-jar-plugin</artifactId>
> <executions>
> <execution>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/pom.xml
> ----------------------------------------------------------------------
> diff --git a/sandbox/prototype/exec/java-exec/pom.xml
> b/sandbox/prototype/exec/java-exec/pom.xml
> index e348bc7..f2f0bff 100644
> --- a/sandbox/prototype/exec/java-exec/pom.xml
> +++ b/sandbox/prototype/exec/java-exec/pom.xml
> @@ -13,14 +13,14 @@
>
> <dependencies>
> <dependency>
> - <groupId>asm</groupId>
> + <groupId>org.ow2.asm</groupId>
> <artifactId>asm-util</artifactId>
> - <version>3.3.1</version>
> + <version>4.1</version>
> </dependency>
> <dependency>
> - <groupId>asm</groupId>
> + <groupId>org.ow2.asm</groupId>
> <artifactId>asm-commons</artifactId>
> - <version>3.3.1</version>
> + <version>4.1</version>
> </dependency>
> <dependency>
> <groupId>com.sun.codemodel</groupId>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> index 079ab6c..4bf6e7e 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> @@ -19,33 +19,37 @@ package org.apache.drill.exec.compile;
>
> import java.io.File;
> import java.io.IOException;
> -import java.io.PrintWriter;
> import java.lang.reflect.Modifier;
> import java.net.URL;
> import java.util.Iterator;
> +import java.util.Random;
> +import java.util.Set;
> import java.util.concurrent.ExecutionException;
> import java.util.concurrent.TimeUnit;
> import java.util.concurrent.atomic.AtomicLong;
>
> import org.apache.drill.exec.exception.ClassTransformationException;
> import org.codehaus.commons.compiler.CompileException;
> -import org.objectweb.asm.ClassAdapter;
> import org.objectweb.asm.ClassReader;
> import org.objectweb.asm.ClassVisitor;
> import org.objectweb.asm.ClassWriter;
> +import org.objectweb.asm.FieldVisitor;
> import org.objectweb.asm.MethodVisitor;
> -import org.objectweb.asm.commons.EmptyVisitor;
> +import org.objectweb.asm.Opcodes;
> +import org.objectweb.asm.commons.Remapper;
> import org.objectweb.asm.commons.RemappingClassAdapter;
> import org.objectweb.asm.commons.RemappingMethodAdapter;
> import org.objectweb.asm.commons.SimpleRemapper;
> import org.objectweb.asm.tree.ClassNode;
> import org.objectweb.asm.tree.FieldNode;
> import org.objectweb.asm.tree.MethodNode;
> -import org.objectweb.asm.util.TraceClassVisitor;
>
> +import com.beust.jcommander.internal.Sets;
> +import com.google.common.base.Preconditions;
> import com.google.common.cache.CacheBuilder;
> import com.google.common.cache.CacheLoader;
> import com.google.common.cache.LoadingCache;
> +import com.google.common.io.Files;
> import com.google.common.io.Resources;
>
> public class ClassTransformer {
> @@ -53,76 +57,113 @@ public class ClassTransformer {
>
> private AtomicLong index = new AtomicLong(0);
> private AtomicLong interfaceIndex = new AtomicLong(0);
> - private LoadingCache<String, byte[]> byteCode =
> CacheBuilder.newBuilder()
> - .maximumSize(10000)
> - .expireAfterWrite(10, TimeUnit.MINUTES)
> - .build(new ClassBytesCacheLoader());
> -
> -
> - private class ClassBytesCacheLoader extends CacheLoader<String, byte[]>{
> + private LoadingCache<String, byte[]> byteCode =
> CacheBuilder.newBuilder().maximumSize(10000)
> + .expireAfterWrite(10, TimeUnit.MINUTES).build(new
> ClassBytesCacheLoader());
> +
> + private class ClassBytesCacheLoader extends CacheLoader<String, byte[]>
> {
> public byte[] load(String path) throws ClassTransformationException,
> IOException {
> URL u = this.getClass().getResource(path);
> - if (u == null) throw new
> ClassTransformationException(String.format("Unable to find TemplateClass at
> path %s",path));
> - return Resources.toByteArray(u);
> + if (u == null)
> + throw new ClassTransformationException(String.format("Unable to
> find TemplateClass at path %s", path));
> + return Resources.toByteArray(u);
> }
> };
> -
> +
> private byte[] getClassByteCodeFromPath(String path) throws
> ClassTransformationException, IOException {
> - try{
> + try {
> return byteCode.get(path);
> } catch (ExecutionException e) {
> Throwable c = e.getCause();
> - if(c instanceof ClassTransformationException) throw
> (ClassTransformationException) c;
> - if(c instanceof IOException) throw (IOException) c;
> + if (c instanceof ClassTransformationException)
> + throw (ClassTransformationException) c;
> + if (c instanceof IOException)
> + throw (IOException) c;
> throw new ClassTransformationException(c);
> }
> }
>
> -
> -
> - @SuppressWarnings("unchecked")
> - public <T, I> T getImplementationClass(QueryClassLoader classLoader,
> - TemplateClassDefinition<T, I> templateDefinition, String
> internalClassBody, I initObject)
> - throws ClassTransformationException {
> - final String implClassName =
> templateDefinition.getTemplateClassName() +
> interfaceIndex.getAndIncrement();
> + public <T, I> T getImplementationClassByBody( //
> + QueryClassLoader classLoader, //
> + TemplateClassDefinition<T> templateDefinition, //
> + String internalClassBody //
> + ) throws ClassTransformationException {
> final String materializedClassName = "org.apache.drill.generated."
> - + templateDefinition.getExternalInterface().getSimpleName() +
> index.getAndIncrement();
> - // final String materializedClassName =
> templateDefinition.getTemplateClassName();
> + + "Gen" +
> templateDefinition.getExternalInterface().getSimpleName() //
> + + index.getAndIncrement();
> + // Get Implementation Class
> try {
> -
> - // Get Implementation Class
> String classBody = ClassBodyBuilder.newBuilder() //
> - .setClassName(implClassName) //
> + .setClassName(materializedClassName) //
>
> .setImplementedInterfaces(templateDefinition.getInternalInterface()) //
> .setBody(internalClassBody) //
> .build();
> - final byte[] implementationClass =
> classLoader.getClassByteCode(implClassName, classBody);
> + return getImplementationClass(classLoader, templateDefinition,
> classBody, materializedClassName);
> + } catch (IOException | CompileException e) {
> + throw new ClassTransformationException("Failure generating class
> body for runtime generated class.", e);
> + }
> +
> + }
> +
> + @SuppressWarnings("unchecked")
> + public <T, I> T getImplementationClass( //
> + QueryClassLoader classLoader, //
> + TemplateClassDefinition<T> templateDefinition, //
> + String entireClass, //
> + String materializedClassName) throws ClassTransformationException {
> +
> + try {
> +
> + final byte[] implementationClass =
> classLoader.getClassByteCode(materializedClassName, entireClass);
>
> // Get Template Class
> final String templateClassName =
> templateDefinition.getTemplateClassName().replaceAll("\\.", File.separator);
> final String templateClassPath = File.separator + templateClassName
> + ".class";
> final byte[] templateClass =
> getClassByteCodeFromPath(templateClassPath);
> -
> + int fileNum = new Random().nextInt(100);
> + Files.write(templateClass, new
> File(String.format("/tmp/%d-template.class", fileNum)));
> // Generate Merge Class
> - ClassNode impl = getClassNodeFromByteCode(implementationClass);
> - // traceClassToSystemOut(implementationClass);
> - // traceClassToSystemOut(templateClass);
> - ClassWriter cw = new ClassWriter(0);
> - MergeAdapter adapter = new MergeAdapter(cw, impl);
> - ClassReader tReader = new ClassReader(templateClass);
> - tReader.accept(adapter, 0);
> -
> - byte[] outputClass = cw.toByteArray();
> -
> - cw = new ClassWriter(0);
> - RemappingClassAdapter r = new RemappingClassAdapter(cw, new
> SimpleRemapper(templateClassName,
> - materializedClassName.replace('.', '/')));
> - new ClassReader(outputClass).accept(r, 0);
> - outputClass = cw.toByteArray();
> - // traceClassToSystemOut(outputClass);
> -
> - // Load the class
> - classLoader.injectByteCode(materializedClassName, outputClass);
> +
> + // Setup adapters for merging, remapping class names and class
> writing. This is done in reverse order of how they
> + // will be evaluated.
> + String oldTemplateSlashName =
> templateDefinition.getTemplateClassName().replace('.', '/');
> + String materializedSlashName = materializedClassName.replace('.',
> '/');
> + RemapClasses remapper = new RemapClasses(oldTemplateSlashName,
> materializedSlashName);
> +
> + {
> + ClassNode impl = getClassNodeFromByteCode(implementationClass);
> + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
> +
> + ClassVisitor remappingAdapter = new RemappingClassAdapter(cw,
> remapper);
> + MergeAdapter mergingAdapter = new
> MergeAdapter(oldTemplateSlashName, materializedSlashName, remappingAdapter,
> + impl);
> + ClassReader tReader = new ClassReader(templateClass);
> + tReader.accept(mergingAdapter, ClassReader.EXPAND_FRAMES);
> + byte[] outputClass = cw.toByteArray();
> + Files.write(outputClass, new
> File(String.format("/tmp/%d-output.class", fileNum)));
> + outputClass = cw.toByteArray();
> +
> + // Load the class
> + classLoader.injectByteCode(materializedClassName, outputClass);
> + }
> +
> + int i = 0;
> + for (String s : remapper.getSubclasses()) {
> + logger.debug("Setting up sub class {}", s);
> + // for each sub class, remap them into the new class.
> + String subclassPath = File.separator + s + ".class";
> + final byte[] bytecode = getClassByteCodeFromPath(subclassPath);
> + RemapClasses localRemapper = new
> RemapClasses(oldTemplateSlashName, materializedSlashName);
> +
> Preconditions.checkArgument(localRemapper.getSubclasses().isEmpty(),
> "Class transformations are only supported for classes that have a single
> level of inner classes.");
> + ClassWriter subcw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
> + ClassVisitor remap = new RemappingClassAdapter(subcw,
> localRemapper);
> + ClassReader reader = new ClassReader(bytecode);
> + reader.accept(remap, ClassReader.EXPAND_FRAMES);
> + byte[] newByteCode = subcw.toByteArray();
> + classLoader.injectByteCode(s.replace(oldTemplateSlashName,
> materializedSlashName).replace('/', '.'), newByteCode);
> + Files.write(subcw.toByteArray(), new
> File(String.format("/tmp/%d-sub-%d.class", fileNum, i)));
> + i++;
> + }
> +
> Class<?> c = classLoader.findClass(materializedClassName);
> if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
> return (T) c.newInstance();
> @@ -131,7 +172,8 @@ public class ClassTransformer {
> }
>
> } catch (CompileException | IOException | ClassNotFoundException |
> InstantiationException | IllegalAccessException e) {
> - throw new ClassTransformationException("Failure generating
> transformation classes.", e);
> + throw new ClassTransformationException(String.format(
> + "Failure generating transformation classes for value: \n %s",
> entireClass), e);
> }
>
> }
> @@ -143,19 +185,37 @@ public class ClassTransformer {
> return impl;
> }
>
> - private void traceClassToSystemOut(byte[] bytecode) {
> - TraceClassVisitor tcv = new TraceClassVisitor(new EmptyVisitor(), new
> PrintWriter(System.out));
> - ClassReader cr = new ClassReader(bytecode);
> - cr.accept(tcv, 0);
> - }
> + // private void traceClassToSystemOut(byte[] bytecode) {
> + // TraceClassVisitor tcv = new TraceClassVisitor(new EmptyVisitor(),
> new PrintWriter(System.out));
> + // ClassReader cr = new ClassReader(bytecode);
> + // cr.accept(tcv, 0);
> + // }
>
> - public class MergeAdapter extends ClassAdapter {
> + public class MergeAdapter extends ClassVisitor {
> private ClassNode classToMerge;
> private String cname;
> + private String templateName;
> + private String newName;
>
> - public MergeAdapter(ClassVisitor cv, ClassNode cn) {
> - super(cv);
> + public MergeAdapter(String templateName, String newName, ClassVisitor
> cv, ClassNode cn) {
> + super(Opcodes.ASM4, cv);
> this.classToMerge = cn;
> + this.templateName = templateName;
> + this.newName = newName.replace('.', '/');
> + ;
> +
> + }
> +
> + @Override
> + public void visitInnerClass(String name, String outerName, String
> innerName, int access) {
> + logger.debug(String.format(
> + "[Inner Class] Name: %s, outerName: %s, innerName: %s,
> templateName: %s, newName: %s.", name, outerName,
> + innerName, templateName, newName));
> + if (name.startsWith(templateName)) {
> + super.visitInnerClass(name.replace(templateName, newName),
> newName, innerName, access);
> + } else {
> + super.visitInnerClass(name, outerName, innerName, access);
> + }
> }
>
> // visit the class
> @@ -193,7 +253,8 @@ public class ClassTransformer {
> MethodNode mn = (MethodNode) it.next();
>
> // skip the init.
> - if (mn.name.equals("<init>")) continue;
> + if (mn.name.equals("<init>"))
> + continue;
>
> String[] exceptions = new String[mn.exceptions.size()];
> mn.exceptions.toArray(exceptions);
> @@ -206,6 +267,47 @@ public class ClassTransformer {
> }
> super.visitEnd();
> }
> +
> + @Override
> + public FieldVisitor visitField(int access, String name, String desc,
> String signature, Object value) {
> + return super.visitField(access, name, desc, signature, value);
> + }
> +
> + }
> +
> + static class RemapClasses extends Remapper {
> +
> + final Set<String> subclasses = Sets.newHashSet();
> +
> + String oldName;
> + String newName;
> +
> + public RemapClasses(String oldName, String newName) {
> + super();
> + Preconditions.checkArgument(!newName.startsWith(oldName), "The new
> name of a class cannot start with the old name of a class, otherwise class
> renaming will cause problems.");
> + this.oldName = oldName;
> + this.newName = newName;
> + }
> +
> + @Override
> + public String map(String typeName) {
> + // remap the names of all classes that start with the old class
> name.
> + if (typeName.startsWith(oldName)) {
> +
> + // write down all the sub classes.
> + if (typeName.startsWith(oldName + "$")){
> + subclasses.add(typeName);
> + }
> +
> + return typeName.replace(oldName, newName);
> + }
> + return typeName;
> + }
> +
> + public Set<String> getSubclasses() {
> + return subclasses;
> + }
> +
> }
>
> -}
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> index 86fe58b..abe2afe 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> @@ -35,15 +35,16 @@ public class JaninoClassCompiler implements
> ClassCompiler{
>
> private IClassLoader compilationClassLoader;
>
> - private boolean debugLines;
> - private boolean debugVars;
> - private boolean debugSource;
> + private boolean debugLines = true;
> + private boolean debugVars = true;
> + private boolean debugSource = true;
>
> public JaninoClassCompiler(ClassLoader parentClassLoader) {
> this.compilationClassLoader = new
> ClassLoaderIClassLoader(parentClassLoader);
> }
>
> public byte[] getClassByteCode(final String className, final String
> code) throws CompileException, IOException, ClassNotFoundException,
> ClassTransformationException {
> + logger.debug("Compiling:\n {}", code);
> StringReader reader = new StringReader(code);
> Scanner scanner = new Scanner((String) null, reader);
> Java.CompilationUnit compilationUnit = new
> Parser(scanner).parseCompilationUnit();
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> index fee4c97..5a01dce 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> @@ -20,21 +20,25 @@ package org.apache.drill.exec.compile;
> import java.lang.reflect.Method;
>
>
> -public class TemplateClassDefinition<T, I>{
> +public class TemplateClassDefinition<T>{
>
> private final Class<T> externalInterface;
> private final String templateClassName;
> private final Class<?> internalInterface;
> - private final Class<I> constructorObject;
> + private final String setupName;
> + private final String evalName;
>
> - public TemplateClassDefinition(Class<T> externalInterface, String
> templateClassName, Class<?> internalInterface, Class<I> constructorObject) {
> +
> + public TemplateClassDefinition(Class<T> externalInterface, String
> templateClassName, Class<?> internalInterface, String setupName, String
> evalName) {
> super();
> this.externalInterface = externalInterface;
> - this.templateClassName = templateClassName;
> + this.templateClassName = templateClassName;
> this.internalInterface = internalInterface;
> - this.constructorObject = constructorObject;
> + this.setupName = setupName;
> + this.evalName = evalName;
> }
>
> +
> public Class<T> getExternalInterface() {
> return externalInterface;
> }
> @@ -48,8 +52,13 @@ public class TemplateClassDefinition<T, I>{
> return templateClassName;
> }
>
> - public Class<I> getConstructorObject() {
> - return constructorObject;
> + public String getSetupName() {
> + return setupName;
> + }
> +
> +
> + public String getEvalName() {
> + return evalName;
> }
>
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
> index 77ae77d..ed6bd9b 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
> @@ -4,13 +4,14 @@ import java.io.IOException;
>
> import org.apache.drill.common.types.TypeProtos.DataMode;
> import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.exec.compile.TemplateClassDefinition;
> import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.BooleanHolder;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.IntHolder;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.LongHolder;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.NullableBooleanHolder;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.NullableIntHolder;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.NullableLongHolder;
> +import org.apache.drill.exec.expr.holders.BooleanHolder;
> +import org.apache.drill.exec.expr.holders.IntHolder;
> +import org.apache.drill.exec.expr.holders.LongHolder;
> +import org.apache.drill.exec.expr.holders.NullableBooleanHolder;
> +import org.apache.drill.exec.expr.holders.NullableIntHolder;
> +import org.apache.drill.exec.expr.holders.NullableLongHolder;
> import org.apache.drill.exec.ops.FragmentContext;
> import org.apache.drill.exec.record.RecordBatch;
>
> @@ -26,7 +27,7 @@ import com.sun.codemodel.JMod;
> import com.sun.codemodel.JType;
> import com.sun.codemodel.JVar;
>
> -public class CodeGenerator {
> +public class CodeGenerator<T> {
> static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(CodeGenerator.class);
>
> public JDefinedClass clazz;
> @@ -37,17 +38,19 @@ public class CodeGenerator {
> private final EvaluationVisitor evaluationVisitor;
> private final String setupName;
> private final String perRecordName;
> -
> + private final TemplateClassDefinition<T> definition;
> private JCodeModel model;
> private int index = 0;
>
> - public CodeGenerator(String setupName, String perRecordName,
> FunctionImplementationRegistry funcRegistry) {
> + public CodeGenerator(TemplateClassDefinition<T> definition,
> FunctionImplementationRegistry funcRegistry) {
> super();
> try{
> - this.setupName = setupName;
> - this.perRecordName = perRecordName;
> + this.definition = definition;
> + this.setupName = definition.getSetupName();
> + this.perRecordName = definition.getEvalName();
> this.model = new JCodeModel();
> this.clazz =
> model._package("org.apache.drill.exec.test.generated")._class("Test1");
> + clazz._implements(definition.getInternalInterface());
> this.parentEvalBlock = new JBlock();
> this.parentSetupBlock = new JBlock();
> this.evaluationVisitor = new EvaluationVisitor(funcRegistry);
> @@ -57,6 +60,7 @@ public class CodeGenerator {
> }
>
> public void addNextWrite(ValueVectorWriteExpression ex){
> + logger.debug("Adding next write {}", ex);
> currentEvalBlock = new JBlock();
> parentEvalBlock.add(currentEvalBlock);
> currentSetupBlock = new JBlock();
> @@ -68,6 +72,10 @@ public class CodeGenerator {
> return currentEvalBlock;
> }
>
> + public String getMaterializedClassName(){
> + return "org.apache.drill.exec.test.generated.Test1";
> + }
> +
> public JBlock getSetupBlock(){
> return currentSetupBlock;
> }
> @@ -134,7 +142,7 @@ public class CodeGenerator {
> }
>
>
> - public class HoldingContainer{
> + public static class HoldingContainer{
> private final JVar holder;
> private final JFieldRef value;
> private final JFieldRef isSet;
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
> new file mode 100644
> index 0000000..5042708
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillBatchFunc.java
> @@ -0,0 +1,10 @@
> +package org.apache.drill.exec.expr;
> +
> +import org.apache.drill.exec.record.RecordBatch;
> +
> +public interface DrillBatchFunc {
> + public void setup(RecordBatch incoming);
> + public void eval();
> + public void batchReset();
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
> index a5bc5fa..04f78d0 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
> @@ -10,20 +10,24 @@ import
> org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
> import org.apache.drill.common.expression.ValueExpressions.LongExpression;
> import org.apache.drill.common.expression.ValueExpressions.QuotedString;
> import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
> +import org.apache.drill.common.types.Types;
> import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
> import org.apache.drill.exec.expr.fn.FunctionHolder;
> import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
> +import
> org.apache.drill.exec.physical.impl.filter.SelectionVectorPopulationExpression;
> +import org.apache.drill.exec.record.vector.SelectionVector2;
> import org.apache.drill.exec.record.vector.TypeHelper;
>
> +import com.google.common.base.Preconditions;
> import com.sun.codemodel.JBlock;
> import com.sun.codemodel.JClass;
> import com.sun.codemodel.JConditional;
> import com.sun.codemodel.JExpr;
> -import com.sun.codemodel.JInvocation;
> +import com.sun.codemodel.JExpression;
> import com.sun.codemodel.JType;
> import com.sun.codemodel.JVar;
>
> -public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, CodeGenerator, RuntimeException> {
> +public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, CodeGenerator<?>, RuntimeException> {
>
> private FunctionImplementationRegistry registry;
>
> @@ -33,7 +37,7 @@ public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, Cod
> }
>
> @Override
> - public HoldingContainer visitFunctionCall(FunctionCall call,
> CodeGenerator generator) throws RuntimeException {
> + public HoldingContainer visitFunctionCall(FunctionCall call,
> CodeGenerator<?> generator) throws RuntimeException {
> HoldingContainer[] args = new HoldingContainer[call.args.size()];
> for(int i = 0; i < call.args.size(); i++){
> args[i] = call.args.get(i).accept(this, generator);
> @@ -43,7 +47,7 @@ public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, Cod
> }
>
> @Override
> - public HoldingContainer visitIfExpression(IfExpression ifExpr,
> CodeGenerator generator) throws RuntimeException {
> + public HoldingContainer visitIfExpression(IfExpression ifExpr,
> CodeGenerator<?> generator) throws RuntimeException {
> JBlock local = generator.getBlock();
>
> HoldingContainer output = generator.declare(ifExpr.getMajorType());
> @@ -93,46 +97,46 @@ public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, Cod
> }
>
> @Override
> - public HoldingContainer visitSchemaPath(SchemaPath path, CodeGenerator
> generator) throws RuntimeException {
> + public HoldingContainer visitSchemaPath(SchemaPath path,
> CodeGenerator<?> generator) throws RuntimeException {
> throw new UnsupportedOperationException("All schema paths should have
> been replaced with ValueVectorExpressions.");
> }
>
> @Override
> - public HoldingContainer visitLongConstant(LongExpression e,
> CodeGenerator generator) throws RuntimeException {
> + public HoldingContainer visitLongConstant(LongExpression e,
> CodeGenerator<?> generator) throws RuntimeException {
> HoldingContainer out = generator.declare(e.getMajorType());
> generator.getBlock().assign(out.getValue(), JExpr.lit(e.getLong()));
> return out;
> }
>
> @Override
> - public HoldingContainer visitDoubleConstant(DoubleExpression e,
> CodeGenerator generator) throws RuntimeException {
> + public HoldingContainer visitDoubleConstant(DoubleExpression e,
> CodeGenerator<?> generator) throws RuntimeException {
> HoldingContainer out = generator.declare(e.getMajorType());
> generator.getBlock().assign(out.getValue(), JExpr.lit(e.getDouble()));
> return out;
> }
>
> @Override
> - public HoldingContainer visitBooleanConstant(BooleanExpression e,
> CodeGenerator generator) throws RuntimeException {
> + public HoldingContainer visitBooleanConstant(BooleanExpression e,
> CodeGenerator<?> generator) throws RuntimeException {
> HoldingContainer out = generator.declare(e.getMajorType());
> generator.getBlock().assign(out.getValue(),
> JExpr.lit(e.getBoolean()));
> return out;
> }
> -
> -
>
> @Override
> - public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator
> generator) throws RuntimeException {
> + public HoldingContainer visitUnknown(LogicalExpression e,
> CodeGenerator<?> generator) throws RuntimeException {
> if(e instanceof ValueVectorReadExpression){
> return visitValueVectorExpression((ValueVectorReadExpression) e,
> generator);
> }else if(e instanceof ValueVectorWriteExpression){
> return visitValueVectorWriteExpression((ValueVectorWriteExpression)
> e, generator);
> + }else if(e instanceof SelectionVectorPopulationExpression){
> + return
> visitSelectionVectorExpression((SelectionVectorPopulationExpression) e,
> generator);
> }else{
> return super.visitUnknown(e, generator);
> }
>
> }
>
> - private HoldingContainer
> visitValueVectorWriteExpression(ValueVectorWriteExpression e, CodeGenerator
> generator){
> + private HoldingContainer
> visitValueVectorWriteExpression(ValueVectorWriteExpression e,
> CodeGenerator<?> generator){
> LogicalExpression child = e.getChild();
> HoldingContainer hc = child.accept(this, generator);
> JBlock block = generator.getBlock();
> @@ -142,9 +146,11 @@ public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, Cod
> JVar vv = generator.declareClassField("vv", vvType);
>
> // get value vector in setup block.
> - generator.getSetupBlock().assign(vv,
> JExpr.direct("outgoing").invoke("getValueVector") //
> - .arg(JExpr.lit(e.getFieldId())) //
> - .arg( ((JClass)vvType).dotclass()));
> + JVar obj = generator.getSetupBlock().decl( //
> + generator.getModel()._ref(Object.class), //
> + generator.getNextVar("obj"), //
> +
> JExpr.direct("outgoing").invoke("getValueVectorById").arg(JExpr.lit(e.getFieldId())).arg(
> ((JClass)vvType).dotclass()));
> + generator.getSetupBlock().assign(vv, JExpr.cast(vvType, obj));
>
> if(hc.isOptional()){
> vv.invoke("set").arg(JExpr.direct("outIndex"));
> @@ -156,18 +162,19 @@ public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, Cod
> return null;
> }
>
> - private HoldingContainer
> visitValueVectorExpression(ValueVectorReadExpression e, CodeGenerator
> generator) throws RuntimeException{
> + private HoldingContainer
> visitValueVectorExpression(ValueVectorReadExpression e, CodeGenerator<?>
> generator) throws RuntimeException{
> // declare value vector
> Class<?> vvClass =
> TypeHelper.getValueVectorClass(e.getMajorType().getMinorType(),
> e.getMajorType().getMode());
> JType vvType = generator.getModel()._ref(vvClass);
> JVar vv1 = generator.declareClassField("vv", vvType);
>
> // get value vector from incoming batch and
> - JInvocation getValue = JExpr //
> - .invoke(JExpr.direct("incoming"), "getValueVector") //
> - .arg(JExpr.lit(e.getFieldId())) //
> - .arg( ((JClass)vvType).dotclass());
> - generator.getSetupBlock().assign(vv1, getValue);
> + // get value vector in setup block.
> + JVar obj = generator.getSetupBlock().decl( //
> + generator.getModel()._ref(Object.class), //
> + generator.getNextVar("obj"), //
> +
> JExpr.direct("outgoing").invoke("getValueVectorById").arg(JExpr.lit(e.getFieldId())).arg(
> ((JClass)vvType).dotclass()));
> + generator.getSetupBlock().assign(vv1, JExpr.cast(vvType, obj));
>
> // evaluation work.
> HoldingContainer out = generator.declare(e.getMajorType());
> @@ -175,25 +182,42 @@ public class EvaluationVisitor extends
> AbstractExprVisitor<HoldingContainer, Cod
>
> if(out.isOptional()){
> JBlock blk = generator.getBlock();
> - blk.assign(out.getIsSet(),
> vv1.invoke("isSet").arg(JExpr.direct("index")));
> + blk.assign(out.getIsSet(),
> vv1.invoke("isSet").arg(JExpr.direct("inIndex")));
> JConditional jc = blk._if(out.getIsSet());
> jc._then() //
> - .assign(out.getValue(),
> vv1.invoke("get").arg(JExpr.direct("index"))); //
> + .assign(out.getValue(),
> vv1.invoke("get").arg(JExpr.direct("inIndex"))); //
> //.assign(out.getIsSet(), JExpr.lit(1));
> //jc._else()
> //.assign(out.getIsSet(), JExpr.lit(0));
>
> }else{
> - generator.getBlock().assign(out.getValue(),
> vv1.invoke("get").arg(JExpr.direct("index")));
> + generator.getBlock().assign(out.getValue(),
> vv1.invoke("get").arg(JExpr.direct("inIndex")));
> }
> return out;
> }
>
>
> + private HoldingContainer
> visitSelectionVectorExpression(SelectionVectorPopulationExpression e,
> CodeGenerator<?> generator){
> + JType svClass = generator.getModel()._ref(SelectionVector2.class);
> + JVar sv = generator.declareClassField("sv", svClass);
> + JVar index = generator.declareClassField("svIndex",
> generator.getModel().CHAR);
> + LogicalExpression child = e.getChild();
> +
> Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN));
> + HoldingContainer hc = child.accept(this, generator);
> + generator.getBlock()._return(hc.getValue());
> +
> +// JBlock blk = generator.getSetupBlock();
> +// blk.assign(sv,
> JExpr.direct("outgoing").invoke("getSelectionVector2"));
> +// JConditional jc = blk._if(hc.getValue());
> +// JBlock body = jc._then();
> +// body.add(sv.invoke("set").arg(index).arg(JExpr.direct("inIndex")));
> +// body.assign(index, index.plus(JExpr.lit(1)));
> + return null;
> + }
>
> @Override
> - public HoldingContainer visitQuotedStringConstant(QuotedString e,
> CodeGenerator CodeGenerator) throws RuntimeException {
> - throw new UnsupportedOperationException("We don't yet support string
> literals as we need to use the valuevector classes and internal vectors.");
> + public HoldingContainer visitQuotedStringConstant(QuotedString e,
> CodeGenerator<?> CodeGenerator) throws RuntimeException {
> + throw new UnsupportedOperationException("We don't yet support string
> literals as we need to build the string value holders.");
> // JExpr stringLiteral = JExpr.lit(e.value);
> //
> CodeGenerator.block.decl(stringLiteral.invoke("getBytes").arg(JExpr.ref(Charsets.UTF_8));
> }
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
> index bc44490..72e5c93 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
> @@ -97,7 +97,7 @@ public class ExpressionTreeMaterializer {
> @Override
> public LogicalExpression visitSchemaPath(SchemaPath path) {
> logger.debug("Visiting schema path {}", path);
> - TypedFieldId tfId = batch.getValueVector(path);
> + TypedFieldId tfId = batch.getValueVectorId(path);
> if (tfId == null) {
> return NullExpression.INSTANCE;
> } else {
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
> index 7276e7d..84f04f0 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
> @@ -16,6 +16,7 @@ import org.apache.drill.exec.expr.annotations.Output;
> import org.apache.drill.exec.expr.annotations.Param;
> import org.apache.drill.exec.expr.annotations.WorkSpace;
> import org.apache.drill.exec.expr.fn.FunctionHolder.ValueReference;
> +import org.apache.drill.exec.expr.fn.FunctionHolder.WorkspaceReference;
> import org.apache.drill.exec.expr.holders.ValueHolder;
> import org.codehaus.commons.compiler.CompileException;
> import org.codehaus.janino.Java;
> @@ -43,7 +44,8 @@ public class FunctionConverter {
>
> // start by getting field information.
> List<ValueReference> params = Lists.newArrayList();
> - List<String> workspaceFields = Lists.newArrayList();
> + List<WorkspaceReference> workspaceFields = Lists.newArrayList();
> +
> ValueReference outputField = null;
>
>
> @@ -96,7 +98,7 @@ public class FunctionConverter {
>
> }else{
> // workspace work.
> - workspaceFields.add(field.getName());
> + workspaceFields.add(new WorkspaceReference(field.getType(),
> field.getName()));
> }
>
> }
> @@ -118,7 +120,8 @@ public class FunctionConverter {
>
> // return holder
> ValueReference[] ps = params.toArray(new
> ValueReference[params.size()]);
> - FunctionHolder fh = new FunctionHolder(template.scope(),
> template.nulls(), template.isBinaryCommutative(), template.name(), ps,
> outputField, methods);
> + WorkspaceReference[] works = workspaceFields.toArray(new
> WorkspaceReference[workspaceFields.size()]);
> + FunctionHolder fh = new FunctionHolder(template.scope(),
> template.nulls(), template.isBinaryCommutative(), template.name(), ps,
> outputField, works, methods);
> return fh;
> }
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
> index 460169f..f7ecd02 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
> @@ -1,5 +1,6 @@
> package org.apache.drill.exec.expr.fn;
>
> +import java.lang.reflect.Type;
> import java.util.Arrays;
> import java.util.Map;
>
> @@ -32,13 +33,15 @@ public class FunctionHolder {
> private String evalBody;
> private String addBody;
> private String setupBody;
> + private WorkspaceReference[] workspaceVars;
> private ValueReference[] parameters;
> private ValueReference returnValue;
>
> - public FunctionHolder(FunctionScope scope, NullHandling nullHandling,
> boolean isBinaryCommutative, String functionName, ValueReference[]
> parameters, ValueReference returnValue, Map<String, String> methods) {
> + public FunctionHolder(FunctionScope scope, NullHandling nullHandling,
> boolean isBinaryCommutative, String functionName, ValueReference[]
> parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
> Map<String, String> methods) {
> super();
> this.scope = scope;
> this.nullHandling = nullHandling;
> + this.workspaceVars = workspaceVars;
> this.isBinaryCommutative = isBinaryCommutative;
> this.functionName = functionName;
> this.setupBody = methods.get("setup");
> @@ -48,9 +51,9 @@ public class FunctionHolder {
> this.returnValue = returnValue;
> }
>
> - public HoldingContainer generateEvalBody(CodeGenerator g,
> HoldingContainer[] inputVariables){
> + public HoldingContainer generateEvalBody(CodeGenerator<?> g,
> HoldingContainer[] inputVariables){
>
> - g.getBlock().directStatement(String.format("//---- start of eval
> portion of %s function. ----//", functionName));
> + //g.getBlock().directStatement(String.format("//---- start of eval
> portion of %s function. ----//", functionName));
>
> JBlock sub = new JBlock(true, true);
>
> @@ -87,6 +90,16 @@ public class FunctionHolder {
> // add the subblock after the out declaration.
> g.getBlock().add(sub);
>
> + JVar[] workspaceJVars = new JVar[workspaceVars.length];
> + for(int i =0 ; i < workspaceVars.length; i++){
> + workspaceJVars[i] = g.declareClassField("work",
> g.getModel()._ref(workspaceVars[i].type)));
> + }
> +
> + for(WorkspaceReference r : workspaceVars){
> + g.declareClassField(, t)
> + }
> +
> + g.declareClassField(prefix, t)
> // locally name external blocks.
>
> // internal out value.
> @@ -99,12 +112,13 @@ public class FunctionHolder {
> sub.decl(JMod.FINAL, inputVariable.getHolder().type(),
> parameter.name, inputVariable.getHolder());
> }
>
> +
> // add function body.
> sub.directStatement(evalBody);
>
> sub.assign(out.getHolder(), internalOutput);
>
> - g.getBlock().directStatement(String.format("//---- end of eval
> portion of %s function. ----//\n", functionName));
> + //g.getBlock().directStatement(String.format("//---- end of eval
> portion of %s function. ----//\n", functionName));
> return out;
> }
>
> @@ -144,6 +158,16 @@ public class FunctionHolder {
>
> }
>
> + public static class WorkspaceReference{
> + Class<?> type;
> + String name;
> + public WorkspaceReference(Class<?> type, String name) {
> + super();
> + this.type = type;
> + this.name = name;
> + }
> +
> + }
> @Override
> public String toString() {
> final int maxLen = 10;
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
> index 702a0ad..939e997 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
> @@ -6,8 +6,8 @@ import
> org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
> import
> org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
> import org.apache.drill.exec.expr.annotations.Output;
> import org.apache.drill.exec.expr.annotations.Param;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.IntHolder;
> -import
> org.apache.drill.exec.expr.holders.ValueHolderImplmenetations.LongHolder;
> +import org.apache.drill.exec.expr.holders.IntHolder;
> +import org.apache.drill.exec.expr.holders.LongHolder;
> import org.apache.drill.exec.record.RecordBatch;
>
> public class MathFunctions{
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
> new file mode 100644
> index 0000000..70bcd10
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/BooleanHolder.java
> @@ -0,0 +1,10 @@
> +package org.apache.drill.exec.expr.holders;
> +
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.common.types.TypeProtos.MinorType;
> +
> +public final class BooleanHolder implements ValueHolder{
> + public static final MajorType TYPE = Types.required(MinorType.BOOLEAN);
> + public int value;
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
> new file mode 100644
> index 0000000..81c82b9
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/IntHolder.java
> @@ -0,0 +1,10 @@
> +package org.apache.drill.exec.expr.holders;
> +
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.common.types.TypeProtos.MinorType;
> +
> +public final class IntHolder implements ValueHolder{
> + public static final MajorType TYPE = Types.required(MinorType.INT);
> + public int value;
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
> new file mode 100644
> index 0000000..c73f9ad
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/LongHolder.java
> @@ -0,0 +1,10 @@
> +package org.apache.drill.exec.expr.holders;
> +
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.common.types.TypeProtos.MinorType;
> +
> +public final class LongHolder implements ValueHolder {
> + public static final MajorType TYPE = Types.required(MinorType.BIGINT);
> + public long value;
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
> new file mode 100644
> index 0000000..f6d2f2e
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableBooleanHolder.java
> @@ -0,0 +1,11 @@
> +package org.apache.drill.exec.expr.holders;
> +
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.common.types.TypeProtos.MinorType;
> +
> +public final class NullableBooleanHolder implements ValueHolder {
> + public static final MajorType TYPE = Types.optional(MinorType.BOOLEAN);
> + public int value;
> + public int isSet;
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
> new file mode 100644
> index 0000000..7eca21e
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableIntHolder.java
> @@ -0,0 +1,11 @@
> +package org.apache.drill.exec.expr.holders;
> +
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.common.types.TypeProtos.MinorType;
> +
> +public final class NullableIntHolder implements ValueHolder {
> + public static final MajorType TYPE = Types.optional(MinorType.INT);
> + public int value;
> + public int isSet;
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
> new file mode 100644
> index 0000000..d1be8f1
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/NullableLongHolder.java
> @@ -0,0 +1,11 @@
> +package org.apache.drill.exec.expr.holders;
> +
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +import org.apache.drill.common.types.TypeProtos.MinorType;
> +
> +public final class NullableLongHolder implements ValueHolder {
> + public static final MajorType TYPE = Types.optional(MinorType.BIGINT);
> + public long value;
> + public int isSet;
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
> deleted file mode 100644
> index adf97fe..0000000
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolderImplmenetations.java
> +++ /dev/null
> @@ -1,44 +0,0 @@
> -package org.apache.drill.exec.expr.holders;
> -
> -import org.apache.drill.common.types.TypeProtos.MajorType;
> -import org.apache.drill.common.types.TypeProtos.MinorType;
> -import org.apache.drill.common.types.Types;
> -
> -public class ValueHolderImplmenetations {
> -
> - private ValueHolderImplmenetations(){}
> -
> - public final static class BooleanHolder implements ValueHolder{
> - public static final MajorType TYPE =
> Types.required(MinorType.BOOLEAN);
> - public int value;
> - }
> -
> - public final static class NullableBooleanHolder implements ValueHolder {
> - public static final MajorType TYPE =
> Types.optional(MinorType.BOOLEAN);
> - public int value;
> - public int isSet;
> - }
> -
> - public final static class IntHolder implements ValueHolder{
> - public static final MajorType TYPE = Types.required(MinorType.INT);
> - public int value;
> - }
> -
> - public final static class NullableIntHolder implements ValueHolder {
> - public static final MajorType TYPE = Types.optional(MinorType.INT);
> - public int value;
> - public int isSet;
> - }
> -
> - public final static class LongHolder implements ValueHolder {
> - public static final MajorType TYPE = Types.required(MinorType.BIGINT);
> - public long value;
> - }
> -
> - public final static class NullableLongHolder implements ValueHolder {
> - public static final MajorType TYPE = Types.optional(MinorType.BIGINT);
> - public long value;
> - public int isSet;
> - }
> -
> -}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> index 1dd7de8..876b873 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> @@ -103,14 +103,11 @@ public class FragmentContext {
> return context.getAllocator();
> }
>
> - public <T> T getImplementationClass(TemplateClassDefinition<T, Void>
> templateDefinition, CodeGenerator cg) throws ClassTransformationException,
> IOException{
> - return transformer.getImplementationClass(this.loader,
> templateDefinition, cg.generate(), null);
> - }
> -
> - public FilteringRecordBatchTransformer
> getFilteringExpression(LogicalExpression expr){
> - return null;
> + public <T> T getImplementationClass(TemplateClassDefinition<T>
> templateDefinition, CodeGenerator<T> cg) throws
> ClassTransformationException, IOException{
> + return transformer.getImplementationClass(this.loader,
> templateDefinition, cg.generate(), cg.getMaterializedClassName());
> }
>
> +
> public void addMetricsToStatus(FragmentStatus.Builder stats){
> stats.setBatchesCompleted(batchesCompleted.get());
> stats.setDataProcessed(dataProcessed.get());
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
> index 80d48f4..1ddc3f9 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
> @@ -78,7 +78,7 @@ public abstract class FilterRecordBatch implements
> RecordBatch {
> }
>
> @Override
> - public TypedFieldId getValueVector(SchemaPath path) {
> + public TypedFieldId getValueVectorId(SchemaPath path) {
> return null;
> }
>
> @@ -111,7 +111,7 @@ public abstract class FilterRecordBatch implements
> RecordBatch {
> IterOutcome o = incoming.next();
> switch (o) {
> case OK_NEW_SCHEMA:
> - transformer = incoming.getContext().getFilteringExpression(null);
> + transformer = null;
> schema = transformer.getSchema();
> // fall through to ok.
> case OK:
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
> index d98c107..739c0d4 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
> @@ -28,9 +28,11 @@ import
> org.apache.drill.exec.physical.base.PhysicalOperator;
> import org.apache.drill.exec.physical.base.Scan;
> import org.apache.drill.exec.physical.config.MockScanBatchCreator;
> import org.apache.drill.exec.physical.config.MockScanPOP;
> +import org.apache.drill.exec.physical.config.Project;
> import org.apache.drill.exec.physical.config.RandomReceiver;
> import org.apache.drill.exec.physical.config.Screen;
> import org.apache.drill.exec.physical.config.SingleSender;
> +import org.apache.drill.exec.physical.impl.project.ProjectBatchCreator;
> import org.apache.drill.exec.record.RecordBatch;
>
> import com.google.common.base.Preconditions;
> @@ -43,6 +45,7 @@ public class ImplCreator extends
> AbstractPhysicalVisitor<RecordBatch, FragmentCo
> private ScreenCreator sc = new ScreenCreator();
> private RandomReceiverCreator rrc = new RandomReceiverCreator();
> private SingleSenderCreator ssc = new SingleSenderCreator();
> + private ProjectBatchCreator pbc = new ProjectBatchCreator();
> private RootExec root = null;
>
> private ImplCreator(){}
> @@ -51,7 +54,11 @@ public class ImplCreator extends
> AbstractPhysicalVisitor<RecordBatch, FragmentCo
> return root;
> }
>
> -
> + @Override
> + public RecordBatch visitProject(Project op, FragmentContext context)
> throws ExecutionSetupException {
> + return pbc.getBatch(context, op, getChildren(op, context));
> + }
> +
> @Override
> public RecordBatch visitScan(Scan<?> scan, FragmentContext context)
> throws ExecutionSetupException {
> Preconditions.checkNotNull(scan);
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
> index 1e0c000..be80349 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
> @@ -173,7 +173,7 @@ public class ScanBatch implements RecordBatch {
> }
>
> @Override
> - public TypedFieldId getValueVector(SchemaPath path) {
> + public TypedFieldId getValueVectorId(SchemaPath path) {
> return holder.getValueVector(path);
> }
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
> index 3819036..516b5af 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
> @@ -50,7 +50,7 @@ public class ScreenCreator implements
> RootCreator<Screen>{
> }
>
>
> - private static class ScreenRoot implements RootExec{
> + static class ScreenRoot implements RootExec{
> static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ScreenRoot.class);
> volatile boolean ok = true;
>
> @@ -138,6 +138,11 @@ public class ScreenCreator implements
> RootCreator<Screen>{
> }
>
> }
> +
> + RecordBatch getIncoming() {
> + return incoming;
> + }
> +
>
> }
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
> index f4921b5..7b2a533 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
> @@ -85,7 +85,7 @@ public class WireRecordBatch implements RecordBatch{
> }
>
> @Override
> - public TypedFieldId getValueVector(SchemaPath path) {
> + public TypedFieldId getValueVectorId(SchemaPath path) {
> return batchLoader.getValueVector(path);
> }
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
> index 158350f..c4a4032 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ExampleFilter.java
> @@ -78,7 +78,7 @@ public class ExampleFilter implements RecordBatch {
> }
>
> @Override
> - public TypedFieldId getValueVector(SchemaPath path) {
> + public TypedFieldId getValueVectorId(SchemaPath path) {
> return null;
> }
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
> new file mode 100644
> index 0000000..d2983ad
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
> @@ -0,0 +1,48 @@
> +package org.apache.drill.exec.physical.impl.filter;
> +
> +import org.apache.drill.exec.record.RecordBatch;
> +import org.apache.drill.exec.record.vector.SelectionVector2;
> +
> +public abstract class FilterTemplate {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FilterTemplate.class);
> +
> + SelectionVector2 outgoingSelectionVector;
> + SelectionVector2 incomingSelectionVector;
> +
> + public void setup(RecordBatch incoming, RecordBatch outgoing){
> + outgoingSelectionVector = outgoing.getSelectionVector2();
> +
> + switch(incoming.getSchema().getSelectionVector()){
> + case NONE:
> + break;
> + case TWO_BYTE:
> + this.incomingSelectionVector = incoming.getSelectionVector2();
> + break;
> + default:
> + throw new UnsupportedOperationException();
> + }
> + }
> +
> + public void filterBatchSV2(int recordCount){
> + int svIndex = 0;
> + for(char i =0; i < recordCount; i++){
> + if(include(i)){
> + outgoingSelectionVector.setIndex(svIndex, i);
> + svIndex+=2;
> + }
> + }
> + }
> +
> + public void filterBatchNoSV(int recordCount){
> + int svIndex = 0;
> + for(char i =0; i < recordCount; i++){
> +
> + if(include(i)){
> + outgoingSelectionVector.setIndex(svIndex, i);
> + svIndex+=2;
> + }
> + }
> + }
> +
> + protected abstract boolean include(int index);
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
> new file mode 100644
> index 0000000..f253695
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/SelectionVectorPopulationExpression.java
> @@ -0,0 +1,39 @@
> +package org.apache.drill.exec.physical.impl.filter;
> +
> +import org.apache.drill.common.expression.ExpressionPosition;
> +import org.apache.drill.common.expression.LogicalExpression;
> +import org.apache.drill.common.expression.visitors.ExprVisitor;
> +import org.apache.drill.common.types.Types;
> +import org.apache.drill.common.types.TypeProtos.MajorType;
> +
> +public class SelectionVectorPopulationExpression implements
> LogicalExpression{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(SelectionVectorPopulationExpression.class);
> +
> + private LogicalExpression child;
> +
> + public SelectionVectorPopulationExpression(LogicalExpression child) {
> + this.child = child;
> + }
> +
> + public LogicalExpression getChild() {
> + return child;
> + }
> +
> + @Override
> + public MajorType getMajorType() {
> + return Types.NULL;
> + }
> +
> + @Override
> + public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E>
> visitor, V value) throws E {
> + return visitor.visitUnknown(this, value);
> + }
> +
> + @Override
> + public ExpressionPosition getPosition() {
> + return ExpressionPosition.UNKNOWN;
> + }
> +
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
> new file mode 100644
> index 0000000..3538469
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectBatchCreator.java
> @@ -0,0 +1,23 @@
> +package org.apache.drill.exec.physical.impl.project;
> +
> +import java.util.List;
> +
> +import org.apache.drill.common.exceptions.ExecutionSetupException;
> +import org.apache.drill.exec.ops.FragmentContext;
> +import org.apache.drill.exec.physical.config.Project;
> +import org.apache.drill.exec.physical.impl.BatchCreator;
> +import org.apache.drill.exec.record.RecordBatch;
> +
> +import com.google.common.base.Preconditions;
> +
> +public class ProjectBatchCreator implements BatchCreator<Project>{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ProjectBatchCreator.class);
> +
> + @Override
> + public RecordBatch getBatch(FragmentContext context, Project config,
> List<RecordBatch> children) throws ExecutionSetupException {
> + Preconditions.checkArgument(children.size() == 1);
> + return new ProjectRecordBatch(config, children.iterator().next(),
> context);
> + }
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
> index 69daae0..86caf28 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java
> @@ -7,6 +7,6 @@ import org.apache.drill.exec.record.RecordBatch;
> public interface ProjectEvaluator {
> static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ProjectEvaluator.class);
>
> - public abstract void setupEvaluators(FragmentContext context,
> RecordBatch incoming) throws SchemaChangeException;
> + public abstract void setupEvaluators(FragmentContext context,
> RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException;
> public abstract void doPerRecordWork(int inIndex, int outIndex);
> }
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
> index cfdb7bc..7a3fec1 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
> @@ -5,6 +5,7 @@ import java.util.List;
>
> import org.apache.drill.common.expression.ErrorCollector;
> import org.apache.drill.common.expression.ErrorCollectorImpl;
> +import org.apache.drill.common.expression.FieldReference;
> import org.apache.drill.common.expression.LogicalExpression;
> import org.apache.drill.common.expression.PathSegment;
> import org.apache.drill.common.expression.SchemaPath;
> @@ -18,6 +19,7 @@ import
> org.apache.drill.exec.expr.ValueVectorReadExpression;
> import org.apache.drill.exec.expr.ValueVectorWriteExpression;
> import org.apache.drill.exec.ops.FragmentContext;
> import org.apache.drill.exec.physical.config.Project;
> +import org.apache.drill.exec.physical.impl.VectorHolder;
> import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
> import org.apache.drill.exec.proto.SchemaDefProtos.NamePart;
> import org.apache.drill.exec.proto.SchemaDefProtos.NamePart.Type;
> @@ -32,6 +34,7 @@ import
> org.apache.drill.exec.record.vector.SelectionVector4;
> import org.apache.drill.exec.record.vector.TypeHelper;
> import org.apache.drill.exec.record.vector.ValueVector;
>
> +
> import com.google.common.base.Preconditions;
> import com.google.common.collect.Lists;
>
> @@ -45,6 +48,7 @@ public class ProjectRecordBatch implements RecordBatch{
> private Projector projector;
> private List<ValueVector<?>> allocationVectors;
> private List<ValueVector<?>> outputVectors;
> + private VectorHolder vh;
>
>
> public ProjectRecordBatch(Project pop, RecordBatch incoming,
> FragmentContext context){
> @@ -86,19 +90,20 @@ public class ProjectRecordBatch implements RecordBatch{
> }
>
> @Override
> - public TypedFieldId getValueVector(SchemaPath path) {
> - return null;
> + public TypedFieldId getValueVectorId(SchemaPath path) {
> + return vh.getValueVector(path);
> }
>
> @Override
> - public <T extends ValueVector<T>> T getValueVectorById(int fieldId,
> Class<?> vvClass) {
> - return null;
> + public <T extends ValueVector<T>> T getValueVectorById(int fieldId,
> Class<?> clazz) {
> + return vh.getValueVector(fieldId, clazz);
> }
>
> @Override
> public IterOutcome next() {
>
> IterOutcome upstream = incoming.next();
> + logger.debug("Upstream... {}", upstream);
> switch(upstream){
> case NONE:
> case NOT_YET:
> @@ -109,6 +114,7 @@ public class ProjectRecordBatch implements RecordBatch{
> projector = createNewProjector();
> }catch(SchemaChangeException ex){
> incoming.kill();
> + logger.error("Failure during query", ex);
> context.fail(ex);
> return IterOutcome.STOP;
> }
> @@ -119,6 +125,9 @@ public class ProjectRecordBatch implements RecordBatch{
> v.allocateNew(recordCount);
> }
> projector.projectRecords(recordCount, 0);
> + for(ValueVector<?> v : this.outputVectors){
> + v.setRecordCount(recordCount);
> + }
> return upstream; // change if upstream changed, otherwise normal.
> default:
> throw new UnsupportedOperationException();
> @@ -134,24 +143,21 @@ public class ProjectRecordBatch implements
> RecordBatch{
> }
> }
> this.outputVectors = Lists.newArrayList();
> -
> + this.vh = new VectorHolder(outputVectors);
> final List<NamedExpression> exprs = pop.getExprs();
> final ErrorCollector collector = new ErrorCollectorImpl();
> final List<TransferPairing<?>> transfers = Lists.newArrayList();
>
> - final CodeGenerator cg = new CodeGenerator("setupEvaluators",
> "doPerRecordWork", context.getFunctionRegistry());
> + final CodeGenerator<Projector> cg = new
> CodeGenerator<Projector>(Projector.TEMPLATE_DEFINITION,
> context.getFunctionRegistry());
>
> for(int i =0; i < exprs.size(); i++){
> final NamedExpression namedExpression = exprs.get(i);
> - final MaterializedField outputField =
> getMaterializedField(namedExpression);
> final LogicalExpression expr =
> ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming,
> collector);
> + final MaterializedField outputField =
> getMaterializedField(namedExpression.getRef(), expr);
> if(collector.hasErrors()){
> throw new SchemaChangeException(String.format("Failure while
> trying to materialize incoming schema. Errors:\n %s.",
> collector.toErrorString()));
> }
>
> -
> -
> -
> // add value vector to transfer if direct reference and this is
> allowed, otherwise, add to evaluation stack.
> if(expr instanceof ValueVectorReadExpression &&
> incoming.getSchema().getSelectionVector() == SelectionVectorMode.NONE){
> ValueVectorReadExpression vectorRead =
> (ValueVectorReadExpression) expr;
> @@ -179,21 +185,21 @@ public class ProjectRecordBatch implements
> RecordBatch{
> this.outSchema = bldr.build();
>
> try {
> - return
> context.getImplementationClass(Projector.TEMPLATE_DEFINITION, cg);
> + Projector projector =
> context.getImplementationClass(Projector.TEMPLATE_DEFINITION, cg);
> + projector.setup(context, incoming, this, transfers);
> + return projector;
> } catch (ClassTransformationException | IOException e) {
> throw new SchemaChangeException("Failure while attempting to load
> generated class", e);
> }
> }
>
> -
> @Override
> public WritableBatch getWritableBatch() {
> - return null;
> + return WritableBatch.get(incoming.getRecordCount(), outputVectors);
> }
>
> -
> - private MaterializedField getMaterializedField(NamedExpression ex){
> - return new MaterializedField(getFieldDef(ex.getRef(),
> ex.getExpr().getMajorType()));
> + private MaterializedField getMaterializedField(FieldReference
> reference, LogicalExpression expr){
> + return new MaterializedField(getFieldDef(reference,
> expr.getMajorType()));
> }
>
> private FieldDef getFieldDef(SchemaPath path, MajorType type){
> @@ -212,7 +218,9 @@ public class ProjectRecordBatch implements RecordBatch{
> }else{
>
> parts.add(NamePart.newBuilder().setType(Type.NAME).setName(seg.getNameSegment().getPath().toString()).build());
> }
> + seg = seg.getChild();
> }
> return parts;
> }
> +
> }
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
> index 31c418c..8f22b40 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java
> @@ -9,12 +9,12 @@ import org.apache.drill.exec.record.RecordBatch;
>
> public interface Projector {
>
> - public abstract void setup(FragmentContext context, RecordBatch
> incoming, List<TransferPairing<?>> transfers) throws SchemaChangeException;
> + public abstract void setup(FragmentContext context, RecordBatch
> incoming, RecordBatch outgoing, List<TransferPairing<?>> transfers)
> throws SchemaChangeException;
>
>
> - public abstract void projectRecords(int recordCount, int
> firstOutputIndex);
> + public abstract int projectRecords(int recordCount, int
> firstOutputIndex);
>
> - public static TemplateClassDefinition<Projector, Void>
> TEMPLATE_DEFINITION = new TemplateClassDefinition<Projector, Void>( //
> - Projector.class,
> "org.apache.drill.exec.physical.impl.project.ProjectTemplate",
> ProjectEvaluator.class, Void.class);
> + public static TemplateClassDefinition<Projector> TEMPLATE_DEFINITION =
> new TemplateClassDefinition<Projector>( //
> + Projector.class,
> "org.apache.drill.exec.physical.impl.project.ProjectorTemplate",
> ProjectEvaluator.class, "setupEvaluators", "doPerRecordWork");
>
> }
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
> index 60af7d2..1163fef 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
> @@ -2,50 +2,28 @@ package org.apache.drill.exec.physical.impl.project;
>
> import java.util.List;
>
> -import org.apache.drill.common.expression.ErrorCollector;
> -import org.apache.drill.common.expression.ErrorCollectorImpl;
> -import org.apache.drill.common.expression.LogicalExpression;
> -import org.apache.drill.common.expression.PathSegment;
> -import org.apache.drill.common.expression.SchemaPath;
> -import org.apache.drill.common.logical.data.NamedExpression;
> -import org.apache.drill.common.types.TypeProtos.MajorType;
> import org.apache.drill.exec.exception.SchemaChangeException;
> -import org.apache.drill.exec.expr.CodeGenerator;
> -import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
> -import org.apache.drill.exec.expr.ValueVectorReadExpression;
> -import org.apache.drill.exec.expr.ValueVectorWriteExpression;
> -import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
> import org.apache.drill.exec.ops.FragmentContext;
> -import org.apache.drill.exec.physical.config.Project;
> -import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
> -import org.apache.drill.exec.proto.SchemaDefProtos.NamePart;
> -import org.apache.drill.exec.proto.SchemaDefProtos.NamePart.Type;
> import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
> -import org.apache.drill.exec.record.MaterializedField;
> import org.apache.drill.exec.record.RecordBatch;
> import org.apache.drill.exec.record.vector.SelectionVector2;
> import org.apache.drill.exec.record.vector.SelectionVector4;
> -import org.apache.drill.exec.record.vector.TypeHelper;
> -import org.apache.drill.exec.record.vector.ValueVector;
>
> -import com.google.common.base.Preconditions;
> import com.google.common.collect.ImmutableList;
> -import com.google.common.collect.Lists;
>
> public abstract class ProjectorTemplate implements Projector {
> static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ProjectorTemplate.class);
> -
> +
> private ImmutableList<TransferPairing<?>> transfers;
> private SelectionVector2 vector2;
> private SelectionVector4 vector4;
> private SelectionVectorMode svMode;
>
> - public ProjectorTemplate(final FragmentContext context, final
> RecordBatch incomingBatch, final Project pop,
> FunctionImplementationRegistry funcRegistry) throws SchemaChangeException{
> - super();
> + public ProjectorTemplate() throws SchemaChangeException{
> }
>
> @Override
> - public final void projectRecords(final int recordCount, int
> firstOutputIndex) {
> + public final int projectRecords(final int recordCount, int
> firstOutputIndex) {
> switch(svMode){
> case FOUR_BYTE:
> throw new UnsupportedOperationException();
> @@ -56,7 +34,7 @@ public abstract class ProjectorTemplate implements
> Projector {
> for(int i = 0; i < count; i+=2, firstOutputIndex++){
> doPerRecordWork(vector2.getIndex(i), firstOutputIndex);
> }
> - return;
> + return recordCount;
>
>
> case NONE:
> @@ -68,7 +46,7 @@ public abstract class ProjectorTemplate implements
> Projector {
> for (int i = 0; i < countN; i++, firstOutputIndex++) {
> doPerRecordWork(i, firstOutputIndex);
> }
> - return;
> + return recordCount;
>
>
> default:
> @@ -77,7 +55,7 @@ public abstract class ProjectorTemplate implements
> Projector {
> }
>
> @Override
> - public final void setup(FragmentContext context, RecordBatch incoming,
> List<TransferPairing<?>> transfers) throws SchemaChangeException{
> + public final void setup(FragmentContext context, RecordBatch incoming,
> RecordBatch outgoing, List<TransferPairing<?>> transfers) throws
> SchemaChangeException{
>
> this.svMode = incoming.getSchema().getSelectionVector();
> switch(svMode){
> @@ -89,10 +67,10 @@ public abstract class ProjectorTemplate implements
> Projector {
> break;
> }
> this.transfers = ImmutableList.copyOf(transfers);
> - setupEvaluators(context, incoming);
> + setupEvaluators(context, incoming, outgoing);
> }
>
> - protected abstract void setupEvaluators(FragmentContext context,
> RecordBatch incoming) throws SchemaChangeException;
> + protected abstract void setupEvaluators(FragmentContext context,
> RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException;
> protected abstract void doPerRecordWork(int inIndex, int outIndex);
>
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
> index 40447ec..0edb93e 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
> @@ -85,10 +85,10 @@ public interface RecordBatch {
> * @param path The path where the vector should be located.
> * @return The local field id associated with this vector.
> */
> - public abstract TypedFieldId getValueVector(SchemaPath path);
> + public abstract TypedFieldId getValueVectorId(SchemaPath path);
>
>
> - public abstract <T extends ValueVector<T>> T getValueVectorById(int
> fieldId, Class<?> vvClass);
> + public abstract <T extends ValueVector<T>> T getValueVectorById(int
> fieldId, Class<?> clazz);
>
> /**
> * Update the data in each Field reading interface for the next range
> of records. Once a RecordBatch returns an
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
> index 20cc82c..0989c1d 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
> @@ -21,6 +21,7 @@ import java.util.List;
> import java.util.Set;
>
> import org.apache.drill.exec.exception.SchemaChangeException;
> +import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
>
> import com.beust.jcommander.internal.Sets;
> import com.google.common.collect.Lists;
> @@ -32,7 +33,7 @@ import com.google.common.collect.Lists;
> public class SchemaBuilder {
> private Set<MaterializedField> fields = Sets.newHashSet();
>
> - private BatchSchema.SelectionVectorMode selectionVectorMode;
> + private BatchSchema.SelectionVectorMode selectionVectorMode =
> SelectionVectorMode.NONE;
>
> SchemaBuilder() {
> }
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/80b1d24f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
> index 3fc39eb..48aa0c2 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
> @@ -32,5 +32,12 @@ public final class NullableFixed8 extends
> NullableValueVector<NullableFixed8, Fi
> return new Fixed8(this.field, allocator);
> }
>
> + public long get(int index){
> + return 1l;
> + }
> +
> + public void set(int index, long value){
> +
> + }
>
> }
>
>