You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/07/20 03:57:25 UTC
[04/53] [abbrv] WIP fragmentation, physical plan, byte compiling,
some vector work
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
new file mode 100644
index 0000000..ac44484
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
@@ -0,0 +1,247 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+/*
+ * Janino - An embedded Java[TM] compiler
+ *
+ * Copyright (c) 2001-2010, Arno Unkrig
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+ * following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the
+ * following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+ * following disclaimer in the documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.CharBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.codehaus.commons.compiler.CompileException;
+import org.codehaus.commons.compiler.IClassBodyEvaluator;
+import org.codehaus.commons.io.MultiReader;
+
+public class ClassBodyBuilder {
+
+ private String[] optionalDefaultImports = null;
+ private String className = IClassBodyEvaluator.DEFAULT_CLASS_NAME;
+ private Class<?> optionalExtendedType = null;
+ private Class<?>[] implementedTypes = new Class[0];
+ private String[] imports = {};
+ private String body;
+ private boolean used = false;
+
+ public static ClassBodyBuilder newBuilder(){
+ return new ClassBodyBuilder();
+ }
+
+ private ClassBodyBuilder(){
+ }
+
+ public ClassBodyBuilder setClassName(String className) {
+ assertNotCooked();
+ this.className = className;
+ return this;
+ }
+
+ public ClassBodyBuilder setDefaultImports(String... optionalDefaultImports) {
+ assertNotCooked();
+ this.optionalDefaultImports = optionalDefaultImports;
+ return this;
+ }
+
+ public ClassBodyBuilder setExtendedClass(Class<?> optionalExtendedType) {
+ assertNotCooked();
+ this.optionalExtendedType = optionalExtendedType;
+ return this;
+ }
+
+ public ClassBodyBuilder setImplementedInterfaces(Class<?>... implementedTypes) {
+ assertNotCooked();
+ this.implementedTypes = implementedTypes;
+ return this;
+ }
+
+ private void assertNotCooked() {
+ assert !used;
+ }
+
+ public ClassBodyBuilder setImports(String[] imports) {
+ assertNotCooked();
+ this.imports = imports;
+ return this;
+ }
+
+ public ClassBodyBuilder setBody(String body) {
+ assertNotCooked();
+ this.body = body;
+ return this;
+ }
+
+ public String build() throws CompileException, IOException {
+ used = true;
+ // Wrap the class body in a compilation unit.
+ {
+ StringWriter sw1 = new StringWriter();
+ {
+ PrintWriter pw = new PrintWriter(sw1);
+
+ // Break the class name up into package name and simple class name.
+ String packageName; // null means default package.
+ String simpleClassName;
+ {
+ int idx = this.className.lastIndexOf('.');
+ if (idx == -1) {
+ packageName = "";
+ simpleClassName = this.className;
+ } else {
+ packageName = this.className.substring(0, idx);
+ simpleClassName = this.className.substring(idx + 1);
+ }
+ }
+
+ // Print PACKAGE directive.
+ if (!packageName.isEmpty()) {
+ pw.print("package ");
+ pw.print(packageName);
+ pw.println(";");
+ }
+
+ // Print default imports.
+ if (this.optionalDefaultImports != null) {
+ for (String defaultImport : this.optionalDefaultImports) {
+ pw.print("import ");
+ pw.print(defaultImport);
+ pw.println(";");
+ }
+ }
+
+ // Print imports as declared in the document.
+ for (String imporT : imports) {
+ pw.print("import ");
+ pw.print(imporT);
+ pw.println(";");
+ }
+
+ // Print the class declaration.
+ pw.print("public class ");
+ pw.print(simpleClassName);
+ if (this.optionalExtendedType != null) {
+ pw.print(" extends ");
+ pw.print(this.optionalExtendedType.getCanonicalName());
+ }
+ if (this.implementedTypes.length > 0) {
+ pw.print(" implements ");
+ pw.print(this.implementedTypes[0].getName());
+ for (int i = 1; i < this.implementedTypes.length; ++i) {
+ pw.print(", ");
+ pw.print(this.implementedTypes[i].getName());
+ }
+ }
+ pw.println(" {");
+ pw.close();
+ }
+
+ StringWriter sw2 = new StringWriter();
+ {
+ PrintWriter pw = new PrintWriter(sw2);
+ pw.println("}");
+ pw.close();
+ }
+
+ return sw1.toString() + body + sw2.toString();
+
+ }
+
+ }
+
+// /**
+// * Heuristically parse IMPORT declarations at the beginning of the character stream produced by the given
+// * {@link Reader}. After this method returns, all characters up to and including that last IMPORT declaration have
+// * been read from the {@link Reader}.
+// * <p>
+// * This method does not handle comments and string literals correctly, i.e. if a pattern that looks like an IMPORT
+// * declaration appears within a comment or a string literal, it will be taken as an IMPORT declaration.
+// *
+// * @param r
+// * A {@link Reader} that supports MARK, e.g. a {@link BufferedReader}
+// * @return The parsed imports, e.g. {@code "java.util.*", "static java.util.Map.Entry" }
+// */
+// protected static String[] parseImportDeclarations(Reader r) throws IOException {
+// final CharBuffer cb = CharBuffer.allocate(10000);
+// r.mark(cb.limit());
+// r.read(cb);
+// cb.rewind();
+//
+// List<String> imports = new ArrayList<String>();
+// int afterLastImport = 0;
+// for (Matcher matcher = IMPORT_STATEMENT_PATTERN.matcher(cb); matcher.find();) {
+// imports.add(matcher.group(1));
+// afterLastImport = matcher.end();
+// }
+// r.reset();
+// r.skip(afterLastImport);
+// return imports.toArray(new String[imports.size()]);
+// }
+//
+// private static final Pattern IMPORT_STATEMENT_PATTERN = Pattern.compile("\\bimport\\s+" + "(" + "(?:static\\s+)?"
+// + "[\\p{javaLowerCase}\\p{javaUpperCase}_\\$][\\p{javaLowerCase}\\p{javaUpperCase}\\d_\\$]*"
+// + "(?:\\.[\\p{javaLowerCase}\\p{javaUpperCase}_\\$][\\p{javaLowerCase}\\p{javaUpperCase}\\d_\\$]*)*"
+// + "(?:\\.\\*)?" + ");");
+
+// @Override
+// public Object createInstance(Reader reader) throws CompileException, IOException {
+// this.cook(reader);
+// try {
+// return this.getClazz().newInstance();
+// } catch (InstantiationException ie) {
+// CompileException ce = new CompileException(
+// ("Class is abstract, an interface, an array class, a primitive type, or void; "
+// + "or has no zero-parameter constructor"), null);
+// ce.initCause(ie);
+// throw ce;
+// } catch (IllegalAccessException iae) {
+// CompileException ce = new CompileException("The class or its zero-parameter constructor is not accessible", null);
+// ce.initCause(iae);
+// throw ce;
+// }
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
new file mode 100644
index 0000000..0fb0115
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
@@ -0,0 +1,29 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+import java.io.IOException;
+
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.codehaus.commons.compiler.CompileException;
+
+interface ClassCompiler {
+
+ public abstract byte[] getClassByteCode(String className, String sourcecode) throws CompileException, IOException, ClassNotFoundException, ClassTransformationException ;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
new file mode 100644
index 0000000..814b239
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
@@ -0,0 +1,210 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Modifier;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.codehaus.commons.compiler.CompileException;
+import org.objectweb.asm.ClassAdapter;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.commons.EmptyVisitor;
+import org.objectweb.asm.commons.RemappingClassAdapter;
+import org.objectweb.asm.commons.RemappingMethodAdapter;
+import org.objectweb.asm.commons.SimpleRemapper;
+import org.objectweb.asm.tree.ClassNode;
+import org.objectweb.asm.tree.FieldNode;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.util.TraceClassVisitor;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.io.Resources;
+
+public class ClassTransformer {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassTransformer.class);
+
+ private AtomicLong index = new AtomicLong(0);
+ private AtomicLong interfaceIndex = new AtomicLong(0);
+ private LoadingCache<String, byte[]> byteCode = CacheBuilder.newBuilder()
+ .maximumSize(10000)
+ .expireAfterWrite(10, TimeUnit.MINUTES)
+ .build(new ClassBytesCacheLoader());
+
+
+ private class ClassBytesCacheLoader extends CacheLoader<String, byte[]>{
+ public byte[] load(String path) throws ClassTransformationException, IOException {
+ URL u = this.getClass().getResource(path);
+ if (u == null) throw new ClassTransformationException(String.format("Unable to find TemplateClass at path %s",path));
+ return Resources.toByteArray(u);
+ }
+ };
+
+ private byte[] getClassByteCodeFromPath(String path) throws ClassTransformationException, IOException {
+ try{
+ return byteCode.get(path);
+ } catch (ExecutionException e) {
+ Throwable c = e.getCause();
+ if(c instanceof ClassTransformationException) throw (ClassTransformationException) c;
+ if(c instanceof IOException) throw (IOException) c;
+ throw new ClassTransformationException(c);
+ }
+ }
+
+
+ @SuppressWarnings("unchecked")
+ public <T, I> T getImplementationClass(QueryClassLoader classLoader,
+ TemplateClassDefinition<T, I> templateDefinition, String internalClassBody, I initObject)
+ throws ClassTransformationException {
+ final String implClassName = templateDefinition.getTemplateClassName() + interfaceIndex.getAndIncrement();
+ final String materializedClassName = "org.apache.drill.generated."
+ + templateDefinition.getExternalInterface().getSimpleName() + index.getAndIncrement();
+ // final String materializedClassName = templateDefinition.getTemplateClassName();
+ try {
+
+ // Get Implementation Class
+ String classBody = ClassBodyBuilder.newBuilder() //
+ .setClassName(implClassName) //
+ .setImplementedInterfaces(templateDefinition.getInternalInterface()) //
+ .setBody(internalClassBody) //
+ .build();
+ final byte[] implementationClass = classLoader.getClassByteCode(implClassName, classBody);
+
+ // Get Template Class
+ final String templateClassName = templateDefinition.getTemplateClassName().replaceAll("\\.", File.separator);
+ final String templateClassPath = File.separator + templateClassName + ".class";
+ final byte[] templateClass = getClassByteCodeFromPath(templateClassPath);
+
+ // Generate Merge Class
+ ClassNode impl = getClassNodeFromByteCode(implementationClass);
+ // traceClassToSystemOut(implementationClass);
+ // traceClassToSystemOut(templateClass);
+ ClassWriter cw = new ClassWriter(0);
+ MergeAdapter adapter = new MergeAdapter(cw, impl);
+ ClassReader tReader = new ClassReader(templateClass);
+ tReader.accept(adapter, 0);
+
+ byte[] outputClass = cw.toByteArray();
+
+ cw = new ClassWriter(0);
+ RemappingClassAdapter r = new RemappingClassAdapter(cw, new SimpleRemapper(templateClassName,
+ materializedClassName.replace('.', '/')));
+ new ClassReader(outputClass).accept(r, 0);
+ outputClass = cw.toByteArray();
+ // traceClassToSystemOut(outputClass);
+
+ // Load the class
+ classLoader.injectByteCode(materializedClassName, outputClass);
+ Class<?> c = classLoader.findClass(materializedClassName);
+ if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
+ return (T) c.newInstance();
+ } else {
+ throw new ClassTransformationException("The requested class did not implement the expected interface.");
+ }
+
+ } catch (CompileException | IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+ throw new ClassTransformationException("Failure generating transformation classes.", e);
+ }
+
+ }
+
+ private ClassNode getClassNodeFromByteCode(byte[] bytes) {
+ ClassReader iReader = new ClassReader(bytes);
+ ClassNode impl = new ClassNode();
+ iReader.accept(impl, 0);
+ return impl;
+ }
+
+ private void traceClassToSystemOut(byte[] bytecode) {
+ TraceClassVisitor tcv = new TraceClassVisitor(new EmptyVisitor(), new PrintWriter(System.out));
+ ClassReader cr = new ClassReader(bytecode);
+ cr.accept(tcv, 0);
+ }
+
+ public class MergeAdapter extends ClassAdapter {
+ private ClassNode classToMerge;
+ private String cname;
+
+ public MergeAdapter(ClassVisitor cv, ClassNode cn) {
+ super(cv);
+ this.classToMerge = cn;
+ }
+
+ // visit the class
+ public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+ // use the access and names of the impl class.
+ super.visit(version, access ^ Modifier.ABSTRACT | Modifier.FINAL, name, signature, superName, interfaces);
+ this.cname = name;
+ }
+
+ @Override
+ public MethodVisitor visitMethod(int access, String arg1, String arg2, String arg3, String[] arg4) {
+ // finalize all methods.
+
+ // skip all abstract methods as they should have implementations.
+ if ((access & Modifier.ABSTRACT) != 0) {
+ // logger.debug("Skipping copy of '{}()' since it is abstract.", arg1);
+ return null;
+ }
+
+ // if( (access & Modifier.PUBLIC) == 0){
+ // access = access ^ Modifier.PUBLIC ^ Modifier.PROTECTED | Modifier.PRIVATE;
+ // }
+ if (!arg1.equals("<init>")) {
+ access = access | Modifier.FINAL;
+ }
+ return super.visitMethod(access, arg1, arg2, arg3, arg4);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void visitEnd() {
+ for (Iterator<?> it = classToMerge.fields.iterator(); it.hasNext();) {
+ ((FieldNode) it.next()).accept(this);
+ }
+ for (Iterator<?> it = classToMerge.methods.iterator(); it.hasNext();) {
+ MethodNode mn = (MethodNode) it.next();
+
+ // skip the init.
+ if (mn.name.equals("<init>")) continue;
+
+ String[] exceptions = new String[mn.exceptions.size()];
+ mn.exceptions.toArray(exceptions);
+ MethodVisitor mv = cv.visitMethod(mn.access | Modifier.FINAL, mn.name, mn.desc, mn.signature, exceptions);
+ mn.instructions.resetLabels();
+ // mn.accept(new RemappingMethodAdapter(mn.access, mn.desc, mv, new
+ // SimpleRemapper("org.apache.drill.exec.compile.ExampleTemplate", "Bunky")));
+ mn.accept(new RemappingMethodAdapter(mn.access, mn.desc, mv, new SimpleRemapper(cname.replace('.', '/'),
+ classToMerge.name.replace('.', '/'))));
+ }
+ super.visitEnd();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
new file mode 100644
index 0000000..15e87fe
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
@@ -0,0 +1,177 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import javax.tools.Diagnostic;
+import javax.tools.DiagnosticListener;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+import javax.tools.JavaCompiler.CompilationTask;
+import javax.tools.JavaFileObject.Kind;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.StandardLocation;
+import javax.tools.ToolProvider;
+
+import org.codehaus.commons.compiler.CompileException;
+import org.codehaus.commons.compiler.Location;
+import org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager;
+import org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager.ByteArrayJavaFileObject;
+
+import com.beust.jcommander.internal.Lists;
+
+class JDKClassCompiler implements ClassCompiler {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JDKClassCompiler.class);
+
+ private boolean debugLines;
+ private boolean debugVars;
+ private boolean debugSource;
+ private Collection<String> compilerOptions = new ArrayList<String>();
+ private DiagnosticListener<JavaFileObject> listener;
+ private final JavaCompiler compiler;
+ private final JavaFileManager fileManager;
+
+ public JDKClassCompiler() {
+ this.compiler = ToolProvider.getSystemJavaCompiler();
+ if (compiler == null) {
+ throw new UnsupportedOperationException(
+ "JDK Java compiler not available - probably you're running a JRE, not a JDK");
+ }
+ this.fileManager = new ByteArrayJavaFileManager<JavaFileManager>(compiler.getStandardFileManager(null, null, null));
+ this.listener = new DiagListener();
+ }
+
+ private JavaFileObject getCompilationUnit(final String s) {
+
+ final URI uri;
+ try {
+ uri = new URI("drill-class-compiler");
+ } catch (URISyntaxException use) {
+ throw new RuntimeException(use);
+ }
+ JavaFileObject javaFileObject = new SimpleJavaFileObject(uri, Kind.SOURCE) {
+
+ @Override
+ public boolean isNameCompatible(String simpleName, Kind kind) {
+ return true;
+ }
+
+ @Override
+ public Reader openReader(boolean ignoreEncodingErrors) throws IOException {
+ return new StringReader(s);
+ }
+
+ @Override
+ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {
+ return s;
+ }
+
+ };
+
+ return javaFileObject;
+ }
+
+ private List<String> getOptions() {
+ List<String> opts = Lists.newArrayList(compilerOptions);
+ String option = this.debugSource ? "-g:source" + (this.debugLines ? ",lines" : "")
+ + (this.debugVars ? ",vars" : "") : this.debugLines ? "-g:lines" + (this.debugVars ? ",vars" : "")
+ : this.debugVars ? "-g:vars" : "-g:none";
+ opts.add(option);
+ return opts;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.drill.exec.compile.ClassCompiler#getClassByteCode(java.lang.String, java.lang.String)
+ */
+ @Override
+ public byte[] getClassByteCode(final String className, final String sourcecode) throws CompileException, IOException,
+ ClassNotFoundException {
+
+ // Create one Java source file in memory, which will be compiled later.
+ JavaFileObject compilationUnit = getCompilationUnit(sourcecode);
+
+ //logger.debug("Compiling the following source code\n{}", sourcecode);
+ // Run the compiler.
+ try {
+ CompilationTask task = compiler.getTask(null, fileManager, listener, getOptions(), null, Collections.singleton(compilationUnit));
+ long n0 = System.nanoTime();
+ if(!task.call()){
+ throw new CompileException("Compilation failed", null);
+ }
+ long n1 = (System.nanoTime() - n0)/1000/1000;
+
+ } catch (RuntimeException rte) {
+
+ // Unwrap the compilation exception and throw it.
+ Throwable cause = rte.getCause();
+ if (cause != null) {
+ cause = cause.getCause();
+ if (cause instanceof CompileException) throw (CompileException) cause;
+ if (cause instanceof IOException) throw (IOException) cause;
+ }
+ throw rte;
+ }
+
+ JavaFileObject classFileObject = fileManager.getJavaFileForInput(StandardLocation.CLASS_OUTPUT, className, Kind.CLASS);
+
+ if (classFileObject == null) {
+ throw new ClassNotFoundException(className + ": Class file not created by compilation");
+ }
+
+ if (!(classFileObject instanceof ByteArrayJavaFileObject))
+ throw new UnsupportedOperationException("Only supports byte array based java file objects.");
+
+ ByteArrayJavaFileObject bajfo = (ByteArrayJavaFileObject) classFileObject;
+ return bajfo.toByteArray();
+
+ }
+
+ private class DiagListener implements DiagnosticListener<JavaFileObject> {
+ @Override
+ public void report(Diagnostic<? extends JavaFileObject> diagnostic) {
+ System.err.println("*** " + diagnostic.toString() + " *** " + diagnostic.getCode());
+
+ Location loc = new Location( //
+ diagnostic.getSource().toString(), //
+ (short) diagnostic.getLineNumber(), //
+ (short) diagnostic.getColumnNumber() //
+ );
+ String code = diagnostic.getCode();
+ String message = diagnostic.getMessage(null) + " (" + code + ")";
+
+ // Wrap the exception in a RuntimeException, because "report()" does not declare checked
+ // exceptions.
+ throw new RuntimeException(new CompileException(message, loc));
+ }
+ }
+
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
new file mode 100644
index 0000000..86fe58b
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.codehaus.commons.compiler.CompileException;
+import org.codehaus.janino.ClassLoaderIClassLoader;
+import org.codehaus.janino.IClassLoader;
+import org.codehaus.janino.Java;
+import org.codehaus.janino.Parser;
+import org.codehaus.janino.Scanner;
+import org.codehaus.janino.UnitCompiler;
+import org.codehaus.janino.util.ClassFile;
+
+public class JaninoClassCompiler implements ClassCompiler{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JaninoClassCompiler.class);
+
+ private IClassLoader compilationClassLoader;
+
+ private boolean debugLines;
+ private boolean debugVars;
+ private boolean debugSource;
+
+ public JaninoClassCompiler(ClassLoader parentClassLoader) {
+ this.compilationClassLoader = new ClassLoaderIClassLoader(parentClassLoader);
+ }
+
+ public byte[] getClassByteCode(final String className, final String code) throws CompileException, IOException, ClassNotFoundException, ClassTransformationException {
+ StringReader reader = new StringReader(code);
+ Scanner scanner = new Scanner((String) null, reader);
+ Java.CompilationUnit compilationUnit = new Parser(scanner).parseCompilationUnit();
+ ClassFile[] classFiles = new UnitCompiler(compilationUnit, compilationClassLoader).compileUnit(this.debugSource,
+ this.debugLines, this.debugVars);
+ if (classFiles.length != 1)
+ throw new ClassTransformationException("Only one class file should have been generated from source code.");
+ return classFiles[0].toByteArray();
+ }
+
+ public void setDebuggingInformation(boolean debugSource, boolean debugLines, boolean debugVars) {
+ this.debugSource = debugSource;
+ this.debugLines = debugLines;
+ this.debugVars = debugVars;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
new file mode 100644
index 0000000..60aa8f3
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+import javax.tools.JavaFileObject.Kind;
+import javax.tools.StandardLocation;
+import javax.tools.ToolProvider;
+
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.codehaus.commons.compiler.CompileException;
+import org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager;
+import org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager.ByteArrayJavaFileObject;
+
+import com.google.common.collect.MapMaker;
+import com.google.common.io.ByteStreams;
+
+public class QueryClassLoader extends URLClassLoader {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryClassLoader.class);
+
+ private final ClassCompiler classCompiler;
+ private ConcurrentMap<String, byte[]> customClasses = new MapMaker().concurrencyLevel(4).makeMap();
+
+ public QueryClassLoader(boolean useJanino) {
+ super(new URL[0]);
+ if (useJanino) {
+ this.classCompiler = new JaninoClassCompiler(this);
+ } else {
+ this.classCompiler = new JDKClassCompiler();
+ }
+ }
+
+ public void injectByteCode(String className, byte[] classBytes) throws IOException {
+ if(customClasses.containsKey(className)) throw new IOException(String.format("The class defined {} has already been loaded.", className));
+ customClasses.put(className, classBytes);
+ }
+
+ @Override
+ protected Class<?> findClass(String className) throws ClassNotFoundException {
+ byte[] ba = customClasses.get(className);
+ if(ba != null){
+ return this.defineClass(className, ba, 0, ba.length);
+ }else{
+ return super.findClass(className);
+ }
+ }
+
+ public byte[] getClassByteCode(final String className, final String sourcecode) throws CompileException, IOException,
+ ClassNotFoundException, ClassTransformationException {
+ byte[] bc = classCompiler.getClassByteCode(className, sourcecode);
+ return bc;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
new file mode 100644
index 0000000..fee4c97
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
@@ -0,0 +1,58 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.compile;
+
+import java.lang.reflect.Method;
+
+
+public class TemplateClassDefinition<T, I>{
+
+ private final Class<T> externalInterface;
+ private final String templateClassName;
+ private final Class<?> internalInterface;
+ private final Class<I> constructorObject;
+
+ public TemplateClassDefinition(Class<T> externalInterface, String templateClassName, Class<?> internalInterface, Class<I> constructorObject) {
+ super();
+ this.externalInterface = externalInterface;
+ this.templateClassName = templateClassName;
+ this.internalInterface = internalInterface;
+ this.constructorObject = constructorObject;
+ }
+
+ public Class<T> getExternalInterface() {
+ return externalInterface;
+ }
+
+
+ public Class<?> getInternalInterface() {
+ return internalInterface;
+ }
+
+ public String getTemplateClassName() {
+ return templateClassName;
+ }
+
+ public Class<I> getConstructorObject() {
+ return constructorObject;
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
index d7ea8fa..d3580b5 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
@@ -17,11 +17,11 @@
******************************************************************************/
package org.apache.drill.exec.coord;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
import java.io.Closeable;
import java.util.Collection;
+import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
+
/**
* Pluggable interface built to manage cluster coordination. Allows Drillbit or DrillClient to register its capabilities
* as well as understand other node's existence and capabilities.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
index 289aa3c..ce0fb92 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
@@ -17,9 +17,9 @@
******************************************************************************/
package org.apache.drill.exec.coord;
+import org.apache.drill.common.proto.CoordinationProtos.DrillServiceInstance;
+import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillServiceInstance;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.netflix.curator.x.discovery.ServiceInstance;
import com.netflix.curator.x.discovery.ServiceInstanceBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
index 3ad08e1..85c573d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
@@ -17,6 +17,17 @@
******************************************************************************/
package org.apache.drill.exec.coord;
+import static com.google.common.base.Throwables.propagate;
+import static com.google.common.collect.Collections2.transform;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.ExecConstants;
+
import com.google.common.base.Function;
import com.netflix.curator.RetryPolicy;
import com.netflix.curator.framework.CuratorFramework;
@@ -28,16 +39,6 @@ import com.netflix.curator.x.discovery.ServiceDiscoveryBuilder;
import com.netflix.curator.x.discovery.ServiceInstance;
import com.netflix.curator.x.discovery.details.ServiceCache;
import com.netflix.curator.x.discovery.details.ServiceCacheListener;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-import static com.google.common.base.Throwables.propagate;
-import static com.google.common.collect.Collections2.transform;
/**
* Manages cluster coordination utilizing zookeeper. *
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
new file mode 100644
index 0000000..13ec95e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
@@ -0,0 +1,47 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.exception;
+
+import org.apache.drill.common.exceptions.DrillException;
+
+public class ClassTransformationException extends DrillException{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassTransformationException.class);
+
+ public ClassTransformationException() {
+ super();
+ }
+
+ public ClassTransformationException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public ClassTransformationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ClassTransformationException(String message) {
+ super(message);
+ }
+
+ public ClassTransformationException(Throwable cause) {
+ super(cause);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
new file mode 100644
index 0000000..c273463
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
@@ -0,0 +1,42 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.exception;
+
+public class FragmentSetupException extends ExecutionSetupException{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentSetupException.class);
+
+ public FragmentSetupException() {
+ super();
+ }
+
+ public FragmentSetupException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public FragmentSetupException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public FragmentSetupException(String message) {
+ super(message);
+ }
+
+ public FragmentSetupException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
new file mode 100644
index 0000000..30e7a63
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
@@ -0,0 +1,22 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.foreman;
+
+public interface CancelableQuery {
+ public boolean cancel(long queryid);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
new file mode 100644
index 0000000..4e4ec77
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
@@ -0,0 +1,24 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.foreman;
+
+public class ExecutionPlanner {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExecutionPlanner.class);
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
new file mode 100644
index 0000000..f138171
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
@@ -0,0 +1,39 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.foreman;
+
+
+public class Foreman extends Thread{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Foreman.class);
+
+ public Foreman(){
+
+ }
+
+ public void doWork(QueryWorkUnit work){
+ // generate fragment structure.
+ // store fragments in distributed grid.
+ // generate any codegen required and store in grid.
+ // drop
+ // do get on the result set you're looking for. Do the initial get on the result node you're looking for. This will return either data or a metadata record set
+ }
+
+ public boolean checkStatus(long queryId){
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
new file mode 100644
index 0000000..bdf4a1e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.foreman;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
+
+import com.google.common.base.Preconditions;
+
+public class QueryWorkUnit {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryWorkUnit.class);
+
+ private PlanFragment rootFragment; // for local
+ private List<PlanFragment> fragments;
+
+ public QueryWorkUnit(PlanFragment rootFragment, List<PlanFragment> fragments) {
+ super();
+ Preconditions.checkNotNull(rootFragment);
+ Preconditions.checkNotNull(fragments);
+ this.rootFragment = rootFragment;
+ this.fragments = fragments;
+ }
+
+ public PlanFragment getRootFragment() {
+ return rootFragment;
+ }
+
+ public List<PlanFragment> getFragments() {
+ return fragments;
+ }
+
+
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
new file mode 100644
index 0000000..96d7d1e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
@@ -0,0 +1,30 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.foreman;
+
+public class ResourceRequest {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResourceRequest.class);
+
+ public long memoryMin;
+ public long memoryDesired;
+
+
+ public static class ResourceAllocation {
+ public long memory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
new file mode 100644
index 0000000..fee6172
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
@@ -0,0 +1,24 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.foreman;
+
+import org.apache.drill.exec.proto.ExecProtos.FragmentStatus;
+
+public interface StatusProvider {
+ public FragmentStatus getStatus();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
new file mode 100644
index 0000000..2b3f574
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -0,0 +1,58 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+
+import java.io.Closeable;
+
+import org.apache.drill.exec.server.DrillbitContext;
+
+/**
+ * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods. Also allows inser
+ */
+public abstract class BufferAllocator implements Closeable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
+
+ /**
+ * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+ * @param size The size in bytes.
+ * @return A new ByteBuf.
+ */
+ public abstract ByteBuf buffer(int size);
+
+ public abstract ByteBufAllocator getUnderlyingAllocator();
+
+ public abstract BufferAllocator getChildAllocator(long initialReservation, long maximumReservation);
+
+ /**
+ * Close and release all buffers generated from this buffer pool.
+ */
+ @Override
+ public abstract void close();
+
+ public static BufferAllocator getAllocator(DrillbitContext context){
+ // TODO: support alternative allocators (including a debugging allocator that records all allocation locations for each buffer).
+ return new DirectBufferAllocator();
+ }
+
+ public abstract long getAllocatedMemory();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
new file mode 100644
index 0000000..8c5b003
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
@@ -0,0 +1,58 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocator;
+
+public class DirectBufferAllocator extends BufferAllocator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DirectBufferAllocator.class);
+
+ private final PooledByteBufAllocator buffer = new PooledByteBufAllocator(true);
+
+ @Override
+ public ByteBuf buffer(int size) {
+ // TODO: wrap it
+ return buffer.directBuffer(size);
+ }
+
+ @Override
+ public long getAllocatedMemory() {
+ return 0;
+ }
+
+ @Override
+ public ByteBufAllocator getUnderlyingAllocator() {
+ return buffer;
+ }
+
+
+
+ @Override
+ public BufferAllocator getChildAllocator(long initialReservation, long maximumReservation) {
+ //TODO: Add child account allocator.
+ return this;
+ }
+
+ @Override
+ public void close() {
+ // TODO: collect all buffers and release them away using a weak hashmap so we don't impact pool work
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
new file mode 100644
index 0000000..6b89c12
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
@@ -0,0 +1,55 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.metrics;
+
+import org.apache.drill.exec.server.DrillbitContext;
+
+import com.yammer.metrics.Counter;
+
+/**
+ * Wraps a parent counter so that local in thread metrics can be collected while collecting for a global counter.
+ */
+public class SingleThreadNestedCounter {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SingleThreadNestedCounter.class);
+
+ private volatile long count;
+ private final Counter counter;
+
+
+ public SingleThreadNestedCounter(DrillbitContext context, String name) {
+ super();
+ this.counter = context.getMetrics().counter(name);
+ }
+
+ public long inc(long n){
+ counter.inc(n);
+ count+= n;
+ return count;
+ }
+
+ public long dec(long n){
+ counter.dec(n);
+ count -= n;
+ return count;
+ }
+
+ public long get(){
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
deleted file mode 100644
index 2ebbef5..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.ops;
-
-import org.apache.drill.exec.record.RecordBatch;
-
-import parquet.schema.MessageType;
-
-public interface BatchIterator {
- static enum IterOutcome{NONE, FULL_NEW_SCHEMA, FULL, PARTIAL_NEW_SCHEMA, PARTIAL, STOP}
- public RecordBatch getBatch();
- public FragmentContext getContext();
- public MessageType getSchema();
- public void kill(QueryOutcome outcome);
- public IterOutcome next();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
new file mode 100644
index 0000000..f626cea
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
@@ -0,0 +1,58 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.vector.SelectionVector;
+
+public abstract class FilteringRecordBatchTransformer {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FilteringRecordBatchTransformer.class);
+
+ final RecordBatch incoming;
+ final SelectionVector selectionVector;
+ final BatchSchema schema;
+
+ public FilteringRecordBatchTransformer(RecordBatch incoming, OutputMutator output, SelectionVector selectionVector) {
+ super();
+ this.incoming = incoming;
+ this.selectionVector = selectionVector;
+ this.schema = innerSetup();
+ }
+
+ public abstract BatchSchema innerSetup();
+
+ /**
+ * Applies the filter to the selection index. Ignores any values in the selection vector, instead creating a.
+ * @return
+ */
+ public abstract int apply();
+
+ /**
+ * Applies the filter to the selection index. Utilizes the existing selection index and only evaluates on those records.
+ * @return
+ */
+ public abstract int applyWithSelection();
+
+ public BatchSchema getSchema() {
+ return schema;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index be1081f..0cf17e9 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -17,18 +17,42 @@
******************************************************************************/
package org.apache.drill.exec.ops;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.metrics.SingleThreadNestedCounter;
+import org.apache.drill.exec.planner.FragmentRunnable;
+import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
import org.apache.drill.exec.rpc.bit.BitCom;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngine;
+import com.yammer.metrics.MetricRegistry;
+import com.yammer.metrics.Timer;
+
+/**
+ * Contextual objects required for execution of a particular fragment.
+ */
public class FragmentContext {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentContext.class);
+ private final static String METRIC_TIMER_FRAGMENT_TIME = MetricRegistry.name(FragmentRunnable.class, "completionTimes");
+ private final static String METRIC_BATCHES_COMPLETED = MetricRegistry.name(FragmentRunnable.class, "batchesCompleted");
+ private final static String METRIC_RECORDS_COMPLETED = MetricRegistry.name(FragmentRunnable.class, "recordsCompleted");
+ private final static String METRIC_DATA_PROCESSED = MetricRegistry.name(FragmentRunnable.class, "dataProcessed");
+
private final DrillbitContext context;
-
- public FragmentContext(DrillbitContext context) {
- this.context = context;
+ private final PlanFragment fragment;
+ public final SingleThreadNestedCounter batchesCompleted;
+ public final SingleThreadNestedCounter recordsCompleted;
+ public final SingleThreadNestedCounter dataProcessed;
+ public final Timer fragmentTime;
+
+ public FragmentContext(DrillbitContext dbContext, PlanFragment fragment) {
+ this.fragmentTime = dbContext.getMetrics().timer(METRIC_TIMER_FRAGMENT_TIME);
+ this.batchesCompleted = new SingleThreadNestedCounter(dbContext, METRIC_BATCHES_COMPLETED);
+ this.recordsCompleted = new SingleThreadNestedCounter(dbContext, METRIC_RECORDS_COMPLETED);
+ this.dataProcessed = new SingleThreadNestedCounter(dbContext, METRIC_DATA_PROCESSED);
+ this.context = dbContext;
+ this.fragment = fragment;
}
public void fail(Throwable cause) {
@@ -39,10 +63,21 @@ public class FragmentContext {
return context;
}
- public StorageEngine getStorageEngine(StorageEngineConfig config){
+ public PlanFragment getFragment() {
+ return fragment;
+ }
+
+ public BufferAllocator getAllocator(){
+ // TODO: A local query allocator to ensure memory limits and accurately gauge memory usage.
+ return context.getAllocator();
+ }
+
+
+ public FilteringRecordBatchTransformer getFilteringExpression(LogicalExpression expr){
return null;
}
+
public BitCom getCommunicator(){
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
new file mode 100644
index 0000000..3c75648
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
@@ -0,0 +1,30 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
+
+public class FragmentConverter {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentConverter.class);
+
+ public static FragmentRoot getFragment(FragmentContext context){
+ PlanFragment m = context.getFragment();
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
new file mode 100644
index 0000000..ddacb41
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
@@ -0,0 +1,37 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+import org.apache.drill.exec.exception.FragmentSetupException;
+
+/**
+ * A FragmentRoot is a node which is the last processing node in a query plan. FragmentTerminals include Exchange
+ * output nodes and storage nodes. They are there driving force behind the completion of a query.
+ */
+public interface FragmentRoot {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentRoot.class);
+
+ /**
+ * Do the next batch of work.
+ * @return Whether or not additional batches of work are necessary.
+ */
+ public boolean next();
+
+
+ public void setup() throws FragmentSetupException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
new file mode 100644
index 0000000..8d4e807
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
@@ -0,0 +1,22 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+public class OperatorFactory {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorFactory.class);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
new file mode 100644
index 0000000..fe37e70
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -0,0 +1,51 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+import java.util.Collection;
+
+import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.server.DrillbitContext;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class QueryContext {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class);
+
+ private long queryId;
+ private DrillbitContext drillbitContext;
+
+ public QueryContext(long queryId, DrillbitContext drllbitContext) {
+ super();
+ this.queryId = queryId;
+ this.drillbitContext = drllbitContext;
+ }
+
+ public long getQueryId() {
+ return queryId;
+ }
+
+ public ObjectMapper getMapper(){
+ return drillbitContext.getConfig().getMapper();
+ }
+
+ public Collection<DrillbitEndpoint> getActiveEndpoints(){
+ return drillbitContext.getBits();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
deleted file mode 100644
index b737f7c..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.ops;
-
-public class QueryOutcome {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryOutcome.class);
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
index 88b8af2..b46804f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
@@ -33,7 +33,7 @@ import com.carrotsearch.hppc.procedures.IntObjectProcedure;
/**
* Record batch used for a particular scan. Operators against one or more
*/
-public class ScanBatch implements RecordBatch {
+public abstract class ScanBatch implements RecordBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanBatch.class);
private IntObjectOpenHashMap<ValueVector<?>> fields = new IntObjectOpenHashMap<ValueVector<?>>();
Re: [04/53] [abbrv] WIP fragmentation, physical plan, byte compiling,
some vector work
Posted by Ted Dunning <te...@gmail.com>.
On Fri, Jul 19, 2013 at 6:57 PM, <ja...@apache.org> wrote:
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
> new file mode 100644
> index 0000000..ac44484
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBodyBuilder.java
> @@ -0,0 +1,247 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +/*
> + * Janino - An embedded Java[TM] compiler
> + *
> + * Copyright (c) 2001-2010, Arno Unkrig
> + * All rights reserved.
> + *
> + * Redistribution and use in source and binary forms, with or without
> modification, are permitted provided that the
> + * following conditions are met:
> + *
> + * 1. Redistributions of source code must retain the above copyright
> notice, this list of conditions and the
> + * following disclaimer.
> + * 2. Redistributions in binary form must reproduce the above
> copyright notice, this list of conditions and the
> + * following disclaimer in the documentation and/or other materials
> provided with the distribution.
> + * 3. The name of the author may not be used to endorse or promote
> products derived from this software without
> + * specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
> IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
> + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
> PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
> + * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
> EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
> + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
> OF USE, DATA, OR PROFITS; OR BUSINESS
> + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
> IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> OF THIS SOFTWARE, EVEN IF ADVISED OF THE
> + * POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +import java.io.BufferedReader;
> +import java.io.IOException;
> +import java.io.PrintWriter;
> +import java.io.Reader;
> +import java.io.StringReader;
> +import java.io.StringWriter;
> +import java.nio.CharBuffer;
> +import java.util.ArrayList;
> +import java.util.List;
> +import java.util.regex.Matcher;
> +import java.util.regex.Pattern;
> +
> +import org.codehaus.commons.compiler.CompileException;
> +import org.codehaus.commons.compiler.IClassBodyEvaluator;
> +import org.codehaus.commons.io.MultiReader;
> +
> +public class ClassBodyBuilder {
> +
> + private String[] optionalDefaultImports = null;
> + private String className = IClassBodyEvaluator.DEFAULT_CLASS_NAME;
> + private Class<?> optionalExtendedType = null;
> + private Class<?>[] implementedTypes = new Class[0];
> + private String[] imports = {};
> + private String body;
> + private boolean used = false;
> +
> + public static ClassBodyBuilder newBuilder(){
> + return new ClassBodyBuilder();
> + }
> +
> + private ClassBodyBuilder(){
> + }
> +
> + public ClassBodyBuilder setClassName(String className) {
> + assertNotCooked();
> + this.className = className;
> + return this;
> + }
> +
> + public ClassBodyBuilder setDefaultImports(String...
> optionalDefaultImports) {
> + assertNotCooked();
> + this.optionalDefaultImports = optionalDefaultImports;
> + return this;
> + }
> +
> + public ClassBodyBuilder setExtendedClass(Class<?> optionalExtendedType)
> {
> + assertNotCooked();
> + this.optionalExtendedType = optionalExtendedType;
> + return this;
> + }
> +
> + public ClassBodyBuilder setImplementedInterfaces(Class<?>...
> implementedTypes) {
> + assertNotCooked();
> + this.implementedTypes = implementedTypes;
> + return this;
> + }
> +
> + private void assertNotCooked() {
> + assert !used;
> + }
> +
> + public ClassBodyBuilder setImports(String[] imports) {
> + assertNotCooked();
> + this.imports = imports;
> + return this;
> + }
> +
> + public ClassBodyBuilder setBody(String body) {
> + assertNotCooked();
> + this.body = body;
> + return this;
> + }
> +
> + public String build() throws CompileException, IOException {
> + used = true;
> + // Wrap the class body in a compilation unit.
> + {
> + StringWriter sw1 = new StringWriter();
> + {
> + PrintWriter pw = new PrintWriter(sw1);
> +
> + // Break the class name up into package name and simple class
> name.
> + String packageName; // null means default package.
> + String simpleClassName;
> + {
> + int idx = this.className.lastIndexOf('.');
> + if (idx == -1) {
> + packageName = "";
> + simpleClassName = this.className;
> + } else {
> + packageName = this.className.substring(0, idx);
> + simpleClassName = this.className.substring(idx + 1);
> + }
> + }
> +
> + // Print PACKAGE directive.
> + if (!packageName.isEmpty()) {
> + pw.print("package ");
> + pw.print(packageName);
> + pw.println(";");
> + }
> +
> + // Print default imports.
> + if (this.optionalDefaultImports != null) {
> + for (String defaultImport : this.optionalDefaultImports) {
> + pw.print("import ");
> + pw.print(defaultImport);
> + pw.println(";");
> + }
> + }
> +
> + // Print imports as declared in the document.
> + for (String imporT : imports) {
> + pw.print("import ");
> + pw.print(imporT);
> + pw.println(";");
> + }
> +
> + // Print the class declaration.
> + pw.print("public class ");
> + pw.print(simpleClassName);
> + if (this.optionalExtendedType != null) {
> + pw.print(" extends ");
> + pw.print(this.optionalExtendedType.getCanonicalName());
> + }
> + if (this.implementedTypes.length > 0) {
> + pw.print(" implements ");
> + pw.print(this.implementedTypes[0].getName());
> + for (int i = 1; i < this.implementedTypes.length; ++i) {
> + pw.print(", ");
> + pw.print(this.implementedTypes[i].getName());
> + }
> + }
> + pw.println(" {");
> + pw.close();
> + }
> +
> + StringWriter sw2 = new StringWriter();
> + {
> + PrintWriter pw = new PrintWriter(sw2);
> + pw.println("}");
> + pw.close();
> + }
> +
> + return sw1.toString() + body + sw2.toString();
> +
> + }
> +
> + }
> +
> +// /**
> +// * Heuristically parse IMPORT declarations at the beginning of the
> character stream produced by the given
> +// * {@link Reader}. After this method returns, all characters up to
> and including that last IMPORT declaration have
> +// * been read from the {@link Reader}.
> +// * <p>
> +// * This method does not handle comments and string literals
> correctly, i.e. if a pattern that looks like an IMPORT
> +// * declaration appears within a comment or a string literal, it will
> be taken as an IMPORT declaration.
> +// *
> +// * @param r
> +// * A {@link Reader} that supports MARK, e.g. a {@link
> BufferedReader}
> +// * @return The parsed imports, e.g. {@code "java.util.*", "static
> java.util.Map.Entry" }
> +// */
> +// protected static String[] parseImportDeclarations(Reader r) throws
> IOException {
> +// final CharBuffer cb = CharBuffer.allocate(10000);
> +// r.mark(cb.limit());
> +// r.read(cb);
> +// cb.rewind();
> +//
> +// List<String> imports = new ArrayList<String>();
> +// int afterLastImport = 0;
> +// for (Matcher matcher = IMPORT_STATEMENT_PATTERN.matcher(cb);
> matcher.find();) {
> +// imports.add(matcher.group(1));
> +// afterLastImport = matcher.end();
> +// }
> +// r.reset();
> +// r.skip(afterLastImport);
> +// return imports.toArray(new String[imports.size()]);
> +// }
> +//
> +// private static final Pattern IMPORT_STATEMENT_PATTERN =
> Pattern.compile("\\bimport\\s+" + "(" + "(?:static\\s+)?"
> +// +
> "[\\p{javaLowerCase}\\p{javaUpperCase}_\\$][\\p{javaLowerCase}\\p{javaUpperCase}\\d_\\$]*"
> +// +
> "(?:\\.[\\p{javaLowerCase}\\p{javaUpperCase}_\\$][\\p{javaLowerCase}\\p{javaUpperCase}\\d_\\$]*)*"
> +// + "(?:\\.\\*)?" + ");");
> +
> +// @Override
> +// public Object createInstance(Reader reader) throws CompileException,
> IOException {
> +// this.cook(reader);
> +// try {
> +// return this.getClazz().newInstance();
> +// } catch (InstantiationException ie) {
> +// CompileException ce = new CompileException(
> +// ("Class is abstract, an interface, an array class, a
> primitive type, or void; "
> +// + "or has no zero-parameter constructor"), null);
> +// ce.initCause(ie);
> +// throw ce;
> +// } catch (IllegalAccessException iae) {
> +// CompileException ce = new CompileException("The class or its
> zero-parameter constructor is not accessible", null);
> +// ce.initCause(iae);
> +// throw ce;
> +// }
> +// }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
> new file mode 100644
> index 0000000..0fb0115
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompiler.java
> @@ -0,0 +1,29 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +import java.io.IOException;
> +
> +import org.apache.drill.exec.exception.ClassTransformationException;
> +import org.codehaus.commons.compiler.CompileException;
> +
> +interface ClassCompiler {
> +
> + public abstract byte[] getClassByteCode(String className, String
> sourcecode) throws CompileException, IOException, ClassNotFoundException,
> ClassTransformationException ;
> +
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> new file mode 100644
> index 0000000..814b239
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
> @@ -0,0 +1,210 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +import java.io.File;
> +import java.io.IOException;
> +import java.io.PrintWriter;
> +import java.lang.reflect.Modifier;
> +import java.net.URL;
> +import java.util.Iterator;
> +import java.util.concurrent.ExecutionException;
> +import java.util.concurrent.TimeUnit;
> +import java.util.concurrent.atomic.AtomicLong;
> +
> +import org.apache.drill.exec.exception.ClassTransformationException;
> +import org.codehaus.commons.compiler.CompileException;
> +import org.objectweb.asm.ClassAdapter;
> +import org.objectweb.asm.ClassReader;
> +import org.objectweb.asm.ClassVisitor;
> +import org.objectweb.asm.ClassWriter;
> +import org.objectweb.asm.MethodVisitor;
> +import org.objectweb.asm.commons.EmptyVisitor;
> +import org.objectweb.asm.commons.RemappingClassAdapter;
> +import org.objectweb.asm.commons.RemappingMethodAdapter;
> +import org.objectweb.asm.commons.SimpleRemapper;
> +import org.objectweb.asm.tree.ClassNode;
> +import org.objectweb.asm.tree.FieldNode;
> +import org.objectweb.asm.tree.MethodNode;
> +import org.objectweb.asm.util.TraceClassVisitor;
> +
> +import com.google.common.cache.CacheBuilder;
> +import com.google.common.cache.CacheLoader;
> +import com.google.common.cache.LoadingCache;
> +import com.google.common.io.Resources;
> +
> +public class ClassTransformer {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ClassTransformer.class);
> +
> + private AtomicLong index = new AtomicLong(0);
> + private AtomicLong interfaceIndex = new AtomicLong(0);
> + private LoadingCache<String, byte[]> byteCode =
> CacheBuilder.newBuilder()
> + .maximumSize(10000)
> + .expireAfterWrite(10, TimeUnit.MINUTES)
> + .build(new ClassBytesCacheLoader());
> +
> +
> + private class ClassBytesCacheLoader extends CacheLoader<String, byte[]>{
> + public byte[] load(String path) throws ClassTransformationException,
> IOException {
> + URL u = this.getClass().getResource(path);
> + if (u == null) throw new
> ClassTransformationException(String.format("Unable to find TemplateClass at
> path %s",path));
> + return Resources.toByteArray(u);
> + }
> + };
> +
> + private byte[] getClassByteCodeFromPath(String path) throws
> ClassTransformationException, IOException {
> + try{
> + return byteCode.get(path);
> + } catch (ExecutionException e) {
> + Throwable c = e.getCause();
> + if(c instanceof ClassTransformationException) throw
> (ClassTransformationException) c;
> + if(c instanceof IOException) throw (IOException) c;
> + throw new ClassTransformationException(c);
> + }
> + }
> +
> +
> + @SuppressWarnings("unchecked")
> + public <T, I> T getImplementationClass(QueryClassLoader classLoader,
> + TemplateClassDefinition<T, I> templateDefinition, String
> internalClassBody, I initObject)
> + throws ClassTransformationException {
> + final String implClassName =
> templateDefinition.getTemplateClassName() +
> interfaceIndex.getAndIncrement();
> + final String materializedClassName = "org.apache.drill.generated."
> + + templateDefinition.getExternalInterface().getSimpleName() +
> index.getAndIncrement();
> + // final String materializedClassName =
> templateDefinition.getTemplateClassName();
> + try {
> +
> + // Get Implementation Class
> + String classBody = ClassBodyBuilder.newBuilder() //
> + .setClassName(implClassName) //
> +
> .setImplementedInterfaces(templateDefinition.getInternalInterface()) //
> + .setBody(internalClassBody) //
> + .build();
> + final byte[] implementationClass =
> classLoader.getClassByteCode(implClassName, classBody);
> +
> + // Get Template Class
> + final String templateClassName =
> templateDefinition.getTemplateClassName().replaceAll("\\.", File.separator);
> + final String templateClassPath = File.separator + templateClassName
> + ".class";
> + final byte[] templateClass =
> getClassByteCodeFromPath(templateClassPath);
> +
> + // Generate Merge Class
> + ClassNode impl = getClassNodeFromByteCode(implementationClass);
> + // traceClassToSystemOut(implementationClass);
> + // traceClassToSystemOut(templateClass);
> + ClassWriter cw = new ClassWriter(0);
> + MergeAdapter adapter = new MergeAdapter(cw, impl);
> + ClassReader tReader = new ClassReader(templateClass);
> + tReader.accept(adapter, 0);
> +
> + byte[] outputClass = cw.toByteArray();
> +
> + cw = new ClassWriter(0);
> + RemappingClassAdapter r = new RemappingClassAdapter(cw, new
> SimpleRemapper(templateClassName,
> + materializedClassName.replace('.', '/')));
> + new ClassReader(outputClass).accept(r, 0);
> + outputClass = cw.toByteArray();
> + // traceClassToSystemOut(outputClass);
> +
> + // Load the class
> + classLoader.injectByteCode(materializedClassName, outputClass);
> + Class<?> c = classLoader.findClass(materializedClassName);
> + if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
> + return (T) c.newInstance();
> + } else {
> + throw new ClassTransformationException("The requested class did
> not implement the expected interface.");
> + }
> +
> + } catch (CompileException | IOException | ClassNotFoundException |
> InstantiationException | IllegalAccessException e) {
> + throw new ClassTransformationException("Failure generating
> transformation classes.", e);
> + }
> +
> + }
> +
> + private ClassNode getClassNodeFromByteCode(byte[] bytes) {
> + ClassReader iReader = new ClassReader(bytes);
> + ClassNode impl = new ClassNode();
> + iReader.accept(impl, 0);
> + return impl;
> + }
> +
> + private void traceClassToSystemOut(byte[] bytecode) {
> + TraceClassVisitor tcv = new TraceClassVisitor(new EmptyVisitor(), new
> PrintWriter(System.out));
> + ClassReader cr = new ClassReader(bytecode);
> + cr.accept(tcv, 0);
> + }
> +
> + public class MergeAdapter extends ClassAdapter {
> + private ClassNode classToMerge;
> + private String cname;
> +
> + public MergeAdapter(ClassVisitor cv, ClassNode cn) {
> + super(cv);
> + this.classToMerge = cn;
> + }
> +
> + // visit the class
> + public void visit(int version, int access, String name, String
> signature, String superName, String[] interfaces) {
> + // use the access and names of the impl class.
> + super.visit(version, access ^ Modifier.ABSTRACT | Modifier.FINAL,
> name, signature, superName, interfaces);
> + this.cname = name;
> + }
> +
> + @Override
> + public MethodVisitor visitMethod(int access, String arg1, String
> arg2, String arg3, String[] arg4) {
> + // finalize all methods.
> +
> + // skip all abstract methods as they should have implementations.
> + if ((access & Modifier.ABSTRACT) != 0) {
> + // logger.debug("Skipping copy of '{}()' since it is abstract.",
> arg1);
> + return null;
> + }
> +
> + // if( (access & Modifier.PUBLIC) == 0){
> + // access = access ^ Modifier.PUBLIC ^ Modifier.PROTECTED |
> Modifier.PRIVATE;
> + // }
> + if (!arg1.equals("<init>")) {
> + access = access | Modifier.FINAL;
> + }
> + return super.visitMethod(access, arg1, arg2, arg3, arg4);
> + }
> +
> + @SuppressWarnings("unchecked")
> + public void visitEnd() {
> + for (Iterator<?> it = classToMerge.fields.iterator();
> it.hasNext();) {
> + ((FieldNode) it.next()).accept(this);
> + }
> + for (Iterator<?> it = classToMerge.methods.iterator();
> it.hasNext();) {
> + MethodNode mn = (MethodNode) it.next();
> +
> + // skip the init.
> + if (mn.name.equals("<init>")) continue;
> +
> + String[] exceptions = new String[mn.exceptions.size()];
> + mn.exceptions.toArray(exceptions);
> + MethodVisitor mv = cv.visitMethod(mn.access | Modifier.FINAL,
> mn.name, mn.desc, mn.signature, exceptions);
> + mn.instructions.resetLabels();
> + // mn.accept(new RemappingMethodAdapter(mn.access, mn.desc, mv,
> new
> + //
> SimpleRemapper("org.apache.drill.exec.compile.ExampleTemplate", "Bunky")));
> + mn.accept(new RemappingMethodAdapter(mn.access, mn.desc, mv, new
> SimpleRemapper(cname.replace('.', '/'),
> + classToMerge.name.replace('.', '/'))));
> + }
> + super.visitEnd();
> + }
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
> new file mode 100644
> index 0000000..15e87fe
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JDKClassCompiler.java
> @@ -0,0 +1,177 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +import java.io.IOException;
> +import java.io.Reader;
> +import java.io.StringReader;
> +import java.net.URI;
> +import java.net.URISyntaxException;
> +import java.util.ArrayList;
> +import java.util.Collection;
> +import java.util.Collections;
> +import java.util.List;
> +
> +import javax.tools.Diagnostic;
> +import javax.tools.DiagnosticListener;
> +import javax.tools.JavaCompiler;
> +import javax.tools.JavaFileManager;
> +import javax.tools.JavaFileObject;
> +import javax.tools.JavaCompiler.CompilationTask;
> +import javax.tools.JavaFileObject.Kind;
> +import javax.tools.SimpleJavaFileObject;
> +import javax.tools.StandardLocation;
> +import javax.tools.ToolProvider;
> +
> +import org.codehaus.commons.compiler.CompileException;
> +import org.codehaus.commons.compiler.Location;
> +import org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager;
> +import
> org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager.ByteArrayJavaFileObject;
> +
> +import com.beust.jcommander.internal.Lists;
> +
> +class JDKClassCompiler implements ClassCompiler {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(JDKClassCompiler.class);
> +
> + private boolean debugLines;
> + private boolean debugVars;
> + private boolean debugSource;
> + private Collection<String> compilerOptions = new ArrayList<String>();
> + private DiagnosticListener<JavaFileObject> listener;
> + private final JavaCompiler compiler;
> + private final JavaFileManager fileManager;
> +
> + public JDKClassCompiler() {
> + this.compiler = ToolProvider.getSystemJavaCompiler();
> + if (compiler == null) {
> + throw new UnsupportedOperationException(
> + "JDK Java compiler not available - probably you're running a
> JRE, not a JDK");
> + }
> + this.fileManager = new
> ByteArrayJavaFileManager<JavaFileManager>(compiler.getStandardFileManager(null,
> null, null));
> + this.listener = new DiagListener();
> + }
> +
> + private JavaFileObject getCompilationUnit(final String s) {
> +
> + final URI uri;
> + try {
> + uri = new URI("drill-class-compiler");
> + } catch (URISyntaxException use) {
> + throw new RuntimeException(use);
> + }
> + JavaFileObject javaFileObject = new SimpleJavaFileObject(uri,
> Kind.SOURCE) {
> +
> + @Override
> + public boolean isNameCompatible(String simpleName, Kind kind) {
> + return true;
> + }
> +
> + @Override
> + public Reader openReader(boolean ignoreEncodingErrors) throws
> IOException {
> + return new StringReader(s);
> + }
> +
> + @Override
> + public CharSequence getCharContent(boolean ignoreEncodingErrors)
> throws IOException {
> + return s;
> + }
> +
> + };
> +
> + return javaFileObject;
> + }
> +
> + private List<String> getOptions() {
> + List<String> opts = Lists.newArrayList(compilerOptions);
> + String option = this.debugSource ? "-g:source" + (this.debugLines ?
> ",lines" : "")
> + + (this.debugVars ? ",vars" : "") : this.debugLines ? "-g:lines"
> + (this.debugVars ? ",vars" : "")
> + : this.debugVars ? "-g:vars" : "-g:none";
> + opts.add(option);
> + return opts;
> + }
> +
> + /* (non-Javadoc)
> + * @see
> org.apache.drill.exec.compile.ClassCompiler#getClassByteCode(java.lang.String,
> java.lang.String)
> + */
> + @Override
> + public byte[] getClassByteCode(final String className, final String
> sourcecode) throws CompileException, IOException,
> + ClassNotFoundException {
> +
> + // Create one Java source file in memory, which will be compiled
> later.
> + JavaFileObject compilationUnit = getCompilationUnit(sourcecode);
> +
> + //logger.debug("Compiling the following source code\n{}", sourcecode);
> + // Run the compiler.
> + try {
> + CompilationTask task = compiler.getTask(null, fileManager,
> listener, getOptions(), null, Collections.singleton(compilationUnit));
> + long n0 = System.nanoTime();
> + if(!task.call()){
> + throw new CompileException("Compilation failed", null);
> + }
> + long n1 = (System.nanoTime() - n0)/1000/1000;
> +
> + } catch (RuntimeException rte) {
> +
> + // Unwrap the compilation exception and throw it.
> + Throwable cause = rte.getCause();
> + if (cause != null) {
> + cause = cause.getCause();
> + if (cause instanceof CompileException) throw (CompileException)
> cause;
> + if (cause instanceof IOException) throw (IOException) cause;
> + }
> + throw rte;
> + }
> +
> + JavaFileObject classFileObject =
> fileManager.getJavaFileForInput(StandardLocation.CLASS_OUTPUT, className,
> Kind.CLASS);
> +
> + if (classFileObject == null) {
> + throw new ClassNotFoundException(className + ": Class file not
> created by compilation");
> + }
> +
> + if (!(classFileObject instanceof ByteArrayJavaFileObject))
> + throw new UnsupportedOperationException("Only supports byte array
> based java file objects.");
> +
> + ByteArrayJavaFileObject bajfo = (ByteArrayJavaFileObject)
> classFileObject;
> + return bajfo.toByteArray();
> +
> + }
> +
> + private class DiagListener implements
> DiagnosticListener<JavaFileObject> {
> + @Override
> + public void report(Diagnostic<? extends JavaFileObject> diagnostic) {
> + System.err.println("*** " + diagnostic.toString() + " *** " +
> diagnostic.getCode());
> +
> + Location loc = new Location( //
> + diagnostic.getSource().toString(), //
> + (short) diagnostic.getLineNumber(), //
> + (short) diagnostic.getColumnNumber() //
> + );
> + String code = diagnostic.getCode();
> + String message = diagnostic.getMessage(null) + " (" + code + ")";
> +
> + // Wrap the exception in a RuntimeException, because "report()"
> does not declare checked
> + // exceptions.
> + throw new RuntimeException(new CompileException(message, loc));
> + }
> + }
> +
> +
> +
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> new file mode 100644
> index 0000000..86fe58b
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java
> @@ -0,0 +1,62 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +import java.io.IOException;
> +import java.io.StringReader;
> +
> +import org.apache.drill.exec.exception.ClassTransformationException;
> +import org.codehaus.commons.compiler.CompileException;
> +import org.codehaus.janino.ClassLoaderIClassLoader;
> +import org.codehaus.janino.IClassLoader;
> +import org.codehaus.janino.Java;
> +import org.codehaus.janino.Parser;
> +import org.codehaus.janino.Scanner;
> +import org.codehaus.janino.UnitCompiler;
> +import org.codehaus.janino.util.ClassFile;
> +
> +public class JaninoClassCompiler implements ClassCompiler{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(JaninoClassCompiler.class);
> +
> + private IClassLoader compilationClassLoader;
> +
> + private boolean debugLines;
> + private boolean debugVars;
> + private boolean debugSource;
> +
> + public JaninoClassCompiler(ClassLoader parentClassLoader) {
> + this.compilationClassLoader = new
> ClassLoaderIClassLoader(parentClassLoader);
> + }
> +
> + public byte[] getClassByteCode(final String className, final String
> code) throws CompileException, IOException, ClassNotFoundException,
> ClassTransformationException {
> + StringReader reader = new StringReader(code);
> + Scanner scanner = new Scanner((String) null, reader);
> + Java.CompilationUnit compilationUnit = new
> Parser(scanner).parseCompilationUnit();
> + ClassFile[] classFiles = new UnitCompiler(compilationUnit,
> compilationClassLoader).compileUnit(this.debugSource,
> + this.debugLines, this.debugVars);
> + if (classFiles.length != 1)
> + throw new ClassTransformationException("Only one class file should
> have been generated from source code.");
> + return classFiles[0].toByteArray();
> + }
> +
> + public void setDebuggingInformation(boolean debugSource, boolean
> debugLines, boolean debugVars) {
> + this.debugSource = debugSource;
> + this.debugLines = debugLines;
> + this.debugVars = debugVars;
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
> new file mode 100644
> index 0000000..60aa8f3
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/QueryClassLoader.java
> @@ -0,0 +1,80 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +import java.io.IOException;
> +import java.io.InputStream;
> +import java.io.OutputStream;
> +import java.net.URL;
> +import java.net.URLClassLoader;
> +import java.util.concurrent.ConcurrentMap;
> +
> +import javax.tools.JavaCompiler;
> +import javax.tools.JavaFileManager;
> +import javax.tools.JavaFileObject;
> +import javax.tools.JavaFileObject.Kind;
> +import javax.tools.StandardLocation;
> +import javax.tools.ToolProvider;
> +
> +import org.apache.drill.exec.exception.ClassTransformationException;
> +import org.codehaus.commons.compiler.CompileException;
> +import org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager;
> +import
> org.codehaus.commons.compiler.jdk.ByteArrayJavaFileManager.ByteArrayJavaFileObject;
> +
> +import com.google.common.collect.MapMaker;
> +import com.google.common.io.ByteStreams;
> +
> +public class QueryClassLoader extends URLClassLoader {
> +
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(QueryClassLoader.class);
> +
> + private final ClassCompiler classCompiler;
> + private ConcurrentMap<String, byte[]> customClasses = new
> MapMaker().concurrencyLevel(4).makeMap();
> +
> + public QueryClassLoader(boolean useJanino) {
> + super(new URL[0]);
> + if (useJanino) {
> + this.classCompiler = new JaninoClassCompiler(this);
> + } else {
> + this.classCompiler = new JDKClassCompiler();
> + }
> + }
> +
> + public void injectByteCode(String className, byte[] classBytes) throws
> IOException {
> + if(customClasses.containsKey(className)) throw new
> IOException(String.format("The class defined {} has already been loaded.",
> className));
> + customClasses.put(className, classBytes);
> + }
> +
> + @Override
> + protected Class<?> findClass(String className) throws
> ClassNotFoundException {
> + byte[] ba = customClasses.get(className);
> + if(ba != null){
> + return this.defineClass(className, ba, 0, ba.length);
> + }else{
> + return super.findClass(className);
> + }
> + }
> +
> + public byte[] getClassByteCode(final String className, final String
> sourcecode) throws CompileException, IOException,
> + ClassNotFoundException, ClassTransformationException {
> + byte[] bc = classCompiler.getClassByteCode(className, sourcecode);
> + return bc;
> +
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> new file mode 100644
> index 0000000..fee4c97
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
> @@ -0,0 +1,58 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.compile;
> +
> +import java.lang.reflect.Method;
> +
> +
> +public class TemplateClassDefinition<T, I>{
> +
> + private final Class<T> externalInterface;
> + private final String templateClassName;
> + private final Class<?> internalInterface;
> + private final Class<I> constructorObject;
> +
> + public TemplateClassDefinition(Class<T> externalInterface, String
> templateClassName, Class<?> internalInterface, Class<I> constructorObject) {
> + super();
> + this.externalInterface = externalInterface;
> + this.templateClassName = templateClassName;
> + this.internalInterface = internalInterface;
> + this.constructorObject = constructorObject;
> + }
> +
> + public Class<T> getExternalInterface() {
> + return externalInterface;
> + }
> +
> +
> + public Class<?> getInternalInterface() {
> + return internalInterface;
> + }
> +
> + public String getTemplateClassName() {
> + return templateClassName;
> + }
> +
> + public Class<I> getConstructorObject() {
> + return constructorObject;
> + }
> +
> +
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
> index d7ea8fa..d3580b5 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
> @@ -17,11 +17,11 @@
>
> ******************************************************************************/
> package org.apache.drill.exec.coord;
>
> -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
> -
> import java.io.Closeable;
> import java.util.Collection;
>
> +import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
> +
> /**
> * Pluggable interface built to manage cluster coordination. Allows
> Drillbit or DrillClient to register its capabilities
> * as well as understand other node's existence and capabilities.
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
> index 289aa3c..ce0fb92 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/DrillServiceInstanceHelper.java
> @@ -17,9 +17,9 @@
>
> ******************************************************************************/
> package org.apache.drill.exec.coord;
>
> +import
> org.apache.drill.common.proto.CoordinationProtos.DrillServiceInstance;
> +import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
> import org.apache.drill.exec.ExecConstants;
> -import
> org.apache.drill.exec.proto.CoordinationProtos.DrillServiceInstance;
> -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
>
> import com.netflix.curator.x.discovery.ServiceInstance;
> import com.netflix.curator.x.discovery.ServiceInstanceBuilder;
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
> index 3ad08e1..85c573d 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
> @@ -17,6 +17,17 @@
>
> ******************************************************************************/
> package org.apache.drill.exec.coord;
>
> +import static com.google.common.base.Throwables.propagate;
> +import static com.google.common.collect.Collections2.transform;
> +
> +import java.io.IOException;
> +import java.util.Collection;
> +import java.util.Collections;
> +
> +import org.apache.drill.common.config.DrillConfig;
> +import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
> +import org.apache.drill.exec.ExecConstants;
> +
> import com.google.common.base.Function;
> import com.netflix.curator.RetryPolicy;
> import com.netflix.curator.framework.CuratorFramework;
> @@ -28,16 +39,6 @@ import
> com.netflix.curator.x.discovery.ServiceDiscoveryBuilder;
> import com.netflix.curator.x.discovery.ServiceInstance;
> import com.netflix.curator.x.discovery.details.ServiceCache;
> import com.netflix.curator.x.discovery.details.ServiceCacheListener;
> -import org.apache.drill.common.config.DrillConfig;
> -import org.apache.drill.exec.ExecConstants;
> -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
> -
> -import java.io.IOException;
> -import java.util.Collection;
> -import java.util.Collections;
> -
> -import static com.google.common.base.Throwables.propagate;
> -import static com.google.common.collect.Collections2.transform;
>
> /**
> * Manages cluster coordination utilizing zookeeper. *
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
> new file mode 100644
> index 0000000..13ec95e
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/ClassTransformationException.java
> @@ -0,0 +1,47 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.exception;
> +
> +import org.apache.drill.common.exceptions.DrillException;
> +
> +public class ClassTransformationException extends DrillException{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ClassTransformationException.class);
> +
> + public ClassTransformationException() {
> + super();
> + }
> +
> + public ClassTransformationException(String message, Throwable cause,
> boolean enableSuppression,
> + boolean writableStackTrace) {
> + super(message, cause, enableSuppression, writableStackTrace);
> + }
> +
> + public ClassTransformationException(String message, Throwable cause) {
> + super(message, cause);
> + }
> +
> + public ClassTransformationException(String message) {
> + super(message);
> + }
> +
> + public ClassTransformationException(Throwable cause) {
> + super(cause);
> + }
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
> new file mode 100644
> index 0000000..c273463
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/exception/FragmentSetupException.java
> @@ -0,0 +1,42 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.exception;
> +
> +public class FragmentSetupException extends ExecutionSetupException{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FragmentSetupException.class);
> +
> + public FragmentSetupException() {
> + super();
> + }
> +
> + public FragmentSetupException(String message, Throwable cause, boolean
> enableSuppression, boolean writableStackTrace) {
> + super(message, cause, enableSuppression, writableStackTrace);
> + }
> +
> + public FragmentSetupException(String message, Throwable cause) {
> + super(message, cause);
> + }
> +
> + public FragmentSetupException(String message) {
> + super(message);
> + }
> +
> + public FragmentSetupException(Throwable cause) {
> + super(cause);
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
> new file mode 100644
> index 0000000..30e7a63
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/CancelableQuery.java
> @@ -0,0 +1,22 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.foreman;
> +
> +public interface CancelableQuery {
> + public boolean cancel(long queryid);
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
> new file mode 100644
> index 0000000..4e4ec77
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ExecutionPlanner.java
> @@ -0,0 +1,24 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.foreman;
> +
> +public class ExecutionPlanner {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ExecutionPlanner.class);
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
> new file mode 100644
> index 0000000..f138171
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/Foreman.java
> @@ -0,0 +1,39 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.foreman;
> +
> +
> +public class Foreman extends Thread{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(Foreman.class);
> +
> + public Foreman(){
> +
> + }
> +
> + public void doWork(QueryWorkUnit work){
> + // generate fragment structure.
> + // store fragments in distributed grid.
> + // generate any codegen required and store in grid.
> + // drop
> + // do get on the result set you're looking for. Do the initial get
> on the result node you're looking for. This will return either data or a
> metadata record set
> + }
> +
> + public boolean checkStatus(long queryId){
> + return false;
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
> new file mode 100644
> index 0000000..bdf4a1e
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/QueryWorkUnit.java
> @@ -0,0 +1,54 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.foreman;
> +
> +import java.util.Collection;
> +import java.util.List;
> +
> +import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
> +
> +import com.google.common.base.Preconditions;
> +
> +public class QueryWorkUnit {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(QueryWorkUnit.class);
> +
> + private PlanFragment rootFragment; // for local
> + private List<PlanFragment> fragments;
> +
> + public QueryWorkUnit(PlanFragment rootFragment, List<PlanFragment>
> fragments) {
> + super();
> + Preconditions.checkNotNull(rootFragment);
> + Preconditions.checkNotNull(fragments);
> + this.rootFragment = rootFragment;
> + this.fragments = fragments;
> + }
> +
> + public PlanFragment getRootFragment() {
> + return rootFragment;
> + }
> +
> + public List<PlanFragment> getFragments() {
> + return fragments;
> + }
> +
> +
> +
> +
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
> new file mode 100644
> index 0000000..96d7d1e
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/ResourceRequest.java
> @@ -0,0 +1,30 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.foreman;
> +
> +public class ResourceRequest {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ResourceRequest.class);
> +
> + public long memoryMin;
> + public long memoryDesired;
> +
> +
> + public static class ResourceAllocation {
> + public long memory;
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
> new file mode 100644
> index 0000000..fee6172
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/foreman/StatusProvider.java
> @@ -0,0 +1,24 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.foreman;
> +
> +import org.apache.drill.exec.proto.ExecProtos.FragmentStatus;
> +
> +public interface StatusProvider {
> + public FragmentStatus getStatus();
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
> new file mode 100644
> index 0000000..2b3f574
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
> @@ -0,0 +1,58 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.memory;
> +
> +import io.netty.buffer.ByteBuf;
> +import io.netty.buffer.ByteBufAllocator;
> +import io.netty.buffer.PooledByteBufAllocator;
> +
> +import java.io.Closeable;
> +
> +import org.apache.drill.exec.server.DrillbitContext;
> +
> +/**
> + * Wrapper class to deal with byte buffer allocation. Ensures users only
> use designated methods. Also allows inser
> + */
> +public abstract class BufferAllocator implements Closeable{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
> +
> + /**
> + * Allocate a new or reused buffer of the provided size. Note that the
> buffer may technically be larger than the requested size for rounding
> purposes. However, the buffers capacity will be set to the configured size.
> + * @param size The size in bytes.
> + * @return A new ByteBuf.
> + */
> + public abstract ByteBuf buffer(int size);
> +
> + public abstract ByteBufAllocator getUnderlyingAllocator();
> +
> + public abstract BufferAllocator getChildAllocator(long
> initialReservation, long maximumReservation);
> +
> + /**
> + * Close and release all buffers generated from this buffer pool.
> + */
> + @Override
> + public abstract void close();
> +
> + public static BufferAllocator getAllocator(DrillbitContext context){
> + // TODO: support alternative allocators (including a debugging
> allocator that records all allocation locations for each buffer).
> + return new DirectBufferAllocator();
> + }
> +
> + public abstract long getAllocatedMemory();
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
> new file mode 100644
> index 0000000..8c5b003
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
> @@ -0,0 +1,58 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.memory;
> +
> +import io.netty.buffer.ByteBuf;
> +import io.netty.buffer.ByteBufAllocator;
> +import io.netty.buffer.PooledByteBufAllocator;
> +
> +public class DirectBufferAllocator extends BufferAllocator{
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(DirectBufferAllocator.class);
> +
> + private final PooledByteBufAllocator buffer = new
> PooledByteBufAllocator(true);
> +
> + @Override
> + public ByteBuf buffer(int size) {
> + // TODO: wrap it
> + return buffer.directBuffer(size);
> + }
> +
> + @Override
> + public long getAllocatedMemory() {
> + return 0;
> + }
> +
> + @Override
> + public ByteBufAllocator getUnderlyingAllocator() {
> + return buffer;
> + }
> +
> +
> +
> + @Override
> + public BufferAllocator getChildAllocator(long initialReservation, long
> maximumReservation) {
> + //TODO: Add child account allocator.
> + return this;
> + }
> +
> + @Override
> + public void close() {
> + // TODO: collect all buffers and release them away using a weak
> hashmap so we don't impact pool work
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
> new file mode 100644
> index 0000000..6b89c12
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
> @@ -0,0 +1,55 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.metrics;
> +
> +import org.apache.drill.exec.server.DrillbitContext;
> +
> +import com.yammer.metrics.Counter;
> +
> +/**
> + * Wraps a parent counter so that local in thread metrics can be
> collected while collecting for a global counter.
> + */
> +public class SingleThreadNestedCounter {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(SingleThreadNestedCounter.class);
> +
> + private volatile long count;
> + private final Counter counter;
> +
> +
> + public SingleThreadNestedCounter(DrillbitContext context, String name) {
> + super();
> + this.counter = context.getMetrics().counter(name);
> + }
> +
> + public long inc(long n){
> + counter.inc(n);
> + count+= n;
> + return count;
> + }
> +
> + public long dec(long n){
> + counter.dec(n);
> + count -= n;
> + return count;
> + }
> +
> + public long get(){
> + return count;
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
> deleted file mode 100644
> index 2ebbef5..0000000
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BatchIterator.java
> +++ /dev/null
> @@ -1,32 +0,0 @@
>
> -/*******************************************************************************
> - * Licensed to the Apache Software Foundation (ASF) under one
> - * or more contributor license agreements. See the NOTICE file
> - * distributed with this work for additional information
> - * regarding copyright ownership. The ASF licenses this file
> - * to you under the Apache License, Version 2.0 (the
> - * "License"); you may not use this file except in compliance
> - * with the License. You may obtain a copy of the License at
> - *
> - * http://www.apache.org/licenses/LICENSE-2.0
> - *
> - * Unless required by applicable law or agreed to in writing, software
> - * distributed under the License is distributed on an "AS IS" BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> - * See the License for the specific language governing permissions and
> - * limitations under the License.
> -
> ******************************************************************************/
> -package org.apache.drill.exec.ops;
> -
> -import org.apache.drill.exec.record.RecordBatch;
> -
> -import parquet.schema.MessageType;
> -
> -public interface BatchIterator {
> - static enum IterOutcome{NONE, FULL_NEW_SCHEMA, FULL,
> PARTIAL_NEW_SCHEMA, PARTIAL, STOP}
> - public RecordBatch getBatch();
> - public FragmentContext getContext();
> - public MessageType getSchema();
> - public void kill(QueryOutcome outcome);
> - public IterOutcome next();
> -
> -}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
> new file mode 100644
> index 0000000..f626cea
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FilteringRecordBatchTransformer.java
> @@ -0,0 +1,58 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.ops;
> +
> +import org.apache.drill.exec.record.BatchSchema;
> +import org.apache.drill.exec.record.RecordBatch;
> +import org.apache.drill.exec.record.vector.SelectionVector;
> +
> +public abstract class FilteringRecordBatchTransformer {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FilteringRecordBatchTransformer.class);
> +
> + final RecordBatch incoming;
> + final SelectionVector selectionVector;
> + final BatchSchema schema;
> +
> + public FilteringRecordBatchTransformer(RecordBatch incoming,
> OutputMutator output, SelectionVector selectionVector) {
> + super();
> + this.incoming = incoming;
> + this.selectionVector = selectionVector;
> + this.schema = innerSetup();
> + }
> +
> + public abstract BatchSchema innerSetup();
> +
> + /**
> + * Applies the filter to the selection index. Ignores any values in
> the selection vector, instead creating a.
> + * @return
> + */
> + public abstract int apply();
> +
> + /**
> + * Applies the filter to the selection index. Utilizes the existing
> selection index and only evaluates on those records.
> + * @return
> + */
> + public abstract int applyWithSelection();
> +
> + public BatchSchema getSchema() {
> + return schema;
> + }
> +
> +
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> index be1081f..0cf17e9 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
> @@ -17,18 +17,42 @@
>
> ******************************************************************************/
> package org.apache.drill.exec.ops;
>
> -import org.apache.drill.common.logical.StorageEngineConfig;
> +import org.apache.drill.common.expression.LogicalExpression;
> +import org.apache.drill.exec.memory.BufferAllocator;
> +import org.apache.drill.exec.metrics.SingleThreadNestedCounter;
> +import org.apache.drill.exec.planner.FragmentRunnable;
> +import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
> import org.apache.drill.exec.rpc.bit.BitCom;
> import org.apache.drill.exec.server.DrillbitContext;
> -import org.apache.drill.exec.store.StorageEngine;
>
> +import com.yammer.metrics.MetricRegistry;
> +import com.yammer.metrics.Timer;
> +
> +/**
> + * Contextual objects required for execution of a particular fragment.
> + */
> public class FragmentContext {
> static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FragmentContext.class);
>
> + private final static String METRIC_TIMER_FRAGMENT_TIME =
> MetricRegistry.name(FragmentRunnable.class, "completionTimes");
> + private final static String METRIC_BATCHES_COMPLETED =
> MetricRegistry.name(FragmentRunnable.class, "batchesCompleted");
> + private final static String METRIC_RECORDS_COMPLETED =
> MetricRegistry.name(FragmentRunnable.class, "recordsCompleted");
> + private final static String METRIC_DATA_PROCESSED =
> MetricRegistry.name(FragmentRunnable.class, "dataProcessed");
> +
> private final DrillbitContext context;
> -
> - public FragmentContext(DrillbitContext context) {
> - this.context = context;
> + private final PlanFragment fragment;
> + public final SingleThreadNestedCounter batchesCompleted;
> + public final SingleThreadNestedCounter recordsCompleted;
> + public final SingleThreadNestedCounter dataProcessed;
> + public final Timer fragmentTime;
> +
> + public FragmentContext(DrillbitContext dbContext, PlanFragment
> fragment) {
> + this.fragmentTime =
> dbContext.getMetrics().timer(METRIC_TIMER_FRAGMENT_TIME);
> + this.batchesCompleted = new SingleThreadNestedCounter(dbContext,
> METRIC_BATCHES_COMPLETED);
> + this.recordsCompleted = new SingleThreadNestedCounter(dbContext,
> METRIC_RECORDS_COMPLETED);
> + this.dataProcessed = new SingleThreadNestedCounter(dbContext,
> METRIC_DATA_PROCESSED);
> + this.context = dbContext;
> + this.fragment = fragment;
> }
>
> public void fail(Throwable cause) {
> @@ -39,10 +63,21 @@ public class FragmentContext {
> return context;
> }
>
> - public StorageEngine getStorageEngine(StorageEngineConfig config){
> + public PlanFragment getFragment() {
> + return fragment;
> + }
> +
> + public BufferAllocator getAllocator(){
> + // TODO: A local query allocator to ensure memory limits and
> accurately gauge memory usage.
> + return context.getAllocator();
> + }
> +
> +
> + public FilteringRecordBatchTransformer
> getFilteringExpression(LogicalExpression expr){
> return null;
> }
>
> +
> public BitCom getCommunicator(){
> return null;
> }
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
> new file mode 100644
> index 0000000..3c75648
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentConverter.java
> @@ -0,0 +1,30 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.ops;
> +
> +import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
> +
> +public class FragmentConverter {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FragmentConverter.class);
> +
> + public static FragmentRoot getFragment(FragmentContext context){
> + PlanFragment m = context.getFragment();
> +
> + return null;
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
> new file mode 100644
> index 0000000..ddacb41
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentRoot.java
> @@ -0,0 +1,37 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.ops;
> +
> +import org.apache.drill.exec.exception.FragmentSetupException;
> +
> +/**
> + * A FragmentRoot is a node which is the last processing node in a query
> plan. FragmentTerminals include Exchange
> + * output nodes and storage nodes. They are there driving force behind
> the completion of a query.
> + */
> +public interface FragmentRoot {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(FragmentRoot.class);
> +
> + /**
> + * Do the next batch of work.
> + * @return Whether or not additional batches of work are necessary.
> + */
> + public boolean next();
> +
> +
> + public void setup() throws FragmentSetupException;
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
> new file mode 100644
> index 0000000..8d4e807
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorFactory.java
> @@ -0,0 +1,22 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.ops;
> +
> +public class OperatorFactory {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(OperatorFactory.class);
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
> new file mode 100644
> index 0000000..fe37e70
> --- /dev/null
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
> @@ -0,0 +1,51 @@
>
> +/*******************************************************************************
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> +
> ******************************************************************************/
> +package org.apache.drill.exec.ops;
> +
> +import java.util.Collection;
> +
> +import org.apache.drill.common.proto.CoordinationProtos.DrillbitEndpoint;
> +import org.apache.drill.exec.server.DrillbitContext;
> +
> +import com.fasterxml.jackson.databind.ObjectMapper;
> +
> +public class QueryContext {
> + static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(QueryContext.class);
> +
> + private long queryId;
> + private DrillbitContext drillbitContext;
> +
> + public QueryContext(long queryId, DrillbitContext drllbitContext) {
> + super();
> + this.queryId = queryId;
> + this.drillbitContext = drllbitContext;
> + }
> +
> + public long getQueryId() {
> + return queryId;
> + }
> +
> + public ObjectMapper getMapper(){
> + return drillbitContext.getConfig().getMapper();
> + }
> +
> + public Collection<DrillbitEndpoint> getActiveEndpoints(){
> + return drillbitContext.getBits();
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
> deleted file mode 100644
> index b737f7c..0000000
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryOutcome.java
> +++ /dev/null
> @@ -1,22 +0,0 @@
>
> -/*******************************************************************************
> - * Licensed to the Apache Software Foundation (ASF) under one
> - * or more contributor license agreements. See the NOTICE file
> - * distributed with this work for additional information
> - * regarding copyright ownership. The ASF licenses this file
> - * to you under the Apache License, Version 2.0 (the
> - * "License"); you may not use this file except in compliance
> - * with the License. You may obtain a copy of the License at
> - *
> - * http://www.apache.org/licenses/LICENSE-2.0
> - *
> - * Unless required by applicable law or agreed to in writing, software
> - * distributed under the License is distributed on an "AS IS" BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> - * See the License for the specific language governing permissions and
> - * limitations under the License.
> -
> ******************************************************************************/
> -package org.apache.drill.exec.ops;
> -
> -public class QueryOutcome {
> - static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(QueryOutcome.class);
> -}
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f0be80dc/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
> ----------------------------------------------------------------------
> diff --git
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
> index 88b8af2..b46804f 100644
> ---
> a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
> +++
> b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ScanBatch.java
> @@ -33,7 +33,7 @@ import
> com.carrotsearch.hppc.procedures.IntObjectProcedure;
> /**
> * Record batch used for a particular scan. Operators against one or more
> */
> -public class ScanBatch implements RecordBatch {
> +public abstract class ScanBatch implements RecordBatch {
> static final org.slf4j.Logger logger =
> org.slf4j.LoggerFactory.getLogger(ScanBatch.class);
>
> private IntObjectOpenHashMap<ValueVector<?>> fields = new
> IntObjectOpenHashMap<ValueVector<?>>();
>
>