You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/04 13:09:47 UTC

[17/52] [abbrv] [partial] DRILL-165: Reorganize directories (moves only)

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java
new file mode 100644
index 0000000..c78980a
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java
@@ -0,0 +1,81 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval;
+
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.ValueReader;
+
+public class IfEvaluator implements BasicEvaluator{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IfEvaluator.class);
+
+  private final IfCond[] conditions;
+  private final BasicEvaluator elseExpression;
+  
+  public IfEvaluator(IfExpression expression, ExprVisitor<BasicEvaluator, Void, RuntimeException> evalBuilder, RecordPointer record){
+    this.conditions = new IfCond[expression.conditions.size()];
+    for(int i =0; i < conditions.length; i++){
+      conditions[i] = new IfCond(expression.conditions.get(i), evalBuilder);
+    }
+    elseExpression = expression.elseExpression.accept(evalBuilder, null);
+  }
+  
+  @Override
+  public DataValue eval() {
+    for(int i = 0 ; i < conditions.length; i++){
+      if(ValueReader.getBoolean(conditions[i].condition.eval())) return conditions[i].valueExpression.eval() ;
+    }
+    return elseExpression.eval();
+  }
+  
+  public class IfCond{
+    private final BasicEvaluator condition;
+    private final BasicEvaluator valueExpression;
+
+    public IfCond(IfCondition c, ExprVisitor<BasicEvaluator, Void, RuntimeException> evalBuilder){
+      this.condition = c.condition.accept(evalBuilder, null);
+      this.valueExpression = c.expression.accept(evalBuilder, null);
+    }
+    
+    public boolean matches(RecordPointer r){
+      return ValueReader.getBoolean(condition.eval());
+    }
+    
+    public DataValue eval(RecordPointer r){
+      return valueExpression.eval();
+    }
+    
+    public boolean isConstant() {
+      return condition.isConstant() && valueExpression.isConstant();
+    }
+  }
+
+  @Override
+  public boolean isConstant() {
+    for(IfCond c : conditions){
+      if(!c.isConstant()) return false;
+    }
+    return elseExpression.isConstant();
+  }
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java
new file mode 100644
index 0000000..5b2c781
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java
@@ -0,0 +1,145 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.parser.ExprLexer;
+import org.apache.drill.common.expression.parser.ExprParser;
+import org.apache.drill.common.expression.visitors.AggregateChecker;
+import org.apache.drill.common.expression.visitors.ConstantChecker;
+import org.apache.drill.common.expression.visitors.SimpleExprVisitor;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.UnbackedRecord;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.eval.fn.FunctionArguments;
+import org.apache.drill.exec.ref.eval.fn.FunctionEvaluatorRegistry;
+import org.apache.drill.exec.ref.exceptions.SetupException;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
+import org.apache.drill.exec.ref.values.ScalarValues.DoubleScalar;
+import org.apache.drill.exec.ref.values.ScalarValues.IntegerScalar;
+import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
+import org.apache.drill.exec.ref.values.ScalarValues.StringScalar;
+
+public class SimpleEvaluationVisitor extends SimpleExprVisitor<BasicEvaluator>{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleEvaluationVisitor.class);
+
+  private RecordPointer record;
+  private List<AggregatingEvaluator> aggregators = new ArrayList<AggregatingEvaluator>();
+  
+  public SimpleEvaluationVisitor(RecordPointer record) {
+    super();
+    this.record = record;
+  }
+
+  public List<AggregatingEvaluator> getAggregators() {
+    return aggregators;
+  }
+
+  @Override
+  public BasicEvaluator visitFunctionCall(FunctionCall call) {
+    List<BasicEvaluator> evals = new ArrayList<BasicEvaluator>();
+    boolean includesAggregates = false;
+    boolean onlyConstants = true;
+    for(LogicalExpression e : call){
+      if(AggregateChecker.isAggregating(e)) includesAggregates = true;
+      if(!ConstantChecker.onlyIncludesConstants(e)) onlyConstants = false;
+      evals.add(e.accept(this, null));
+    }
+    FunctionArguments args = new FunctionArguments(onlyConstants, includesAggregates, evals, call);
+
+    if(call.getDefinition().isAggregating()){
+      if(args.includesAggregates()) throw new SetupException(String.format("The call for %s contains one or more arguments that also contain aggregating functions.  An aggregating function cannot contain aggregating expressions.", call.getDefinition()));
+      BasicEvaluator e = FunctionEvaluatorRegistry.getEvaluator(call.getDefinition().getName(), args, record);
+      if(!(e instanceof AggregatingEvaluator ) ){
+        throw new SetupException(String.format("Function %s is an aggregating function.  However, the provided evaluator (%s) is not an aggregating evaluator.", call.getDefinition(), e.getClass()));
+      }
+      
+      aggregators.add( (AggregatingEvaluator) e);
+      return e;
+    }else{
+      BasicEvaluator e = FunctionEvaluatorRegistry.getEvaluator(call.getDefinition().getName(), args, record);
+      return e;
+    }
+  }
+
+  @Override
+  public BasicEvaluator visitIfExpression(IfExpression ifExpr) {
+    return new IfEvaluator(ifExpr, this, record);
+  }
+
+  @Override
+  public BasicEvaluator visitSchemaPath(SchemaPath path) {
+    return new FieldEvaluator(path, record);
+  }
+
+  @Override
+  public BasicEvaluator visitLongConstant(LongExpression longExpr) {
+    return new LongScalar(longExpr.getLong());
+  }
+
+  @Override
+  public BasicEvaluator visitDoubleConstant(DoubleExpression dExpr) {
+    return new DoubleScalar(dExpr.getDouble());
+  }
+
+  @Override
+  public BasicEvaluator visitBooleanConstant(BooleanExpression e) {
+    return new BooleanScalar(e.getBoolean());
+  }
+
+  @Override
+  public BasicEvaluator visitQuotedStringConstant(QuotedString e) {
+    return new StringScalar(e.value);
+  }
+  
+  
+  @Override
+  public BasicEvaluator visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
+    throw new UnsupportedOperationException();
+  }
+
+  public static void main(String[] args) throws Exception {
+    String expr = "if( a == 1) then 4 else 2 end";
+    ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+    CommonTokenStream tokens = new CommonTokenStream(lexer);
+    ExprParser parser = new ExprParser(tokens);
+    LogicalExpression e = parser.parse().e;
+    RecordPointer r = new UnbackedRecord();
+    r.addField(new SchemaPath("a", ExpressionPosition.UNKNOWN), new IntegerScalar(3));
+    SimpleEvaluationVisitor builder = new SimpleEvaluationVisitor(r);
+    BasicEvaluator eval = e.accept(builder, null);
+    DataValue v = eval.eval();
+    System.out.println(v);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java
new file mode 100644
index 0000000..3e6d428
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java
@@ -0,0 +1,137 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn;
+
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.BaseBasicEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.exceptions.RecordException;
+import org.apache.drill.exec.ref.values.ComparableValue;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
+
+public class ComparisonEvaluators {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparisonEvaluators.class);
+  
+  @FunctionEvaluator("equal")
+  public static class EqualEvaluator extends BaseBasicEvaluator{
+    private final BasicEvaluator left;
+    private final BasicEvaluator right;
+    
+    public EqualEvaluator(RecordPointer record, FunctionArguments args){
+      super(args.isOnlyConstants(), record);
+      left = args.getEvaluator(0);
+      right = args.getEvaluator(1);
+    }
+    
+    @Override
+    public BooleanScalar eval() {
+      return new BooleanScalar(left.eval().equals(right.eval()));
+    }
+
+  }
+
+  public static boolean isComparable(DataValue a, DataValue b) {
+      return a instanceof ComparableValue && b instanceof ComparableValue && ((ComparableValue) a).supportsCompare(b);
+  }
+  
+  private abstract static class ComparisonEvaluator extends BaseBasicEvaluator{
+    private final BasicEvaluator left;
+    private final BasicEvaluator right;
+    
+    public ComparisonEvaluator(RecordPointer record, FunctionArguments args){
+      super(args.isOnlyConstants(), record);
+      left = args.getEvaluator(0);
+      right = args.getEvaluator(1);
+    }
+    
+    public abstract boolean valid(int i);
+    
+    @Override
+    public BooleanScalar eval() {
+      DataValue a = left.eval();
+      DataValue b = right.eval();
+      
+      if(isComparable(a, b)){
+        int i = ((ComparableValue)a).compareTo(b);
+        return new BooleanScalar(valid( i));
+      }else{
+        throw new RecordException(String.format("Values cannot be compared.  A %s cannot be compared to a %s.", a, b), null);
+      }
+    }
+  }
+  
+  @FunctionEvaluator("less than")
+  public static class LessThan extends ComparisonEvaluator{
+
+    public LessThan(RecordPointer record, FunctionArguments args) {
+      super(record, args);
+    }
+
+    @Override
+    public boolean valid(int i) {
+      return i == -1;
+    }
+    
+  }
+  
+  @FunctionEvaluator("greater than")
+  public static class GreaterThan extends ComparisonEvaluator{
+
+    public GreaterThan(RecordPointer record, FunctionArguments args) {
+      super(record, args);
+    }
+
+    @Override
+    public boolean valid(int i) {
+      return i == 1;
+    }
+    
+  }
+  
+  @FunctionEvaluator("greater than or equal to")
+  public static class GreaterOrEqualTo extends ComparisonEvaluator{
+
+    public GreaterOrEqualTo(RecordPointer record, FunctionArguments args) {
+      super(record, args);
+    }
+
+    @Override
+    public boolean valid(int i) {
+      return i >= 0;
+    }
+    
+  }
+  
+  @FunctionEvaluator("less than or equal to")
+  public static class LessThanOrEqualTo extends ComparisonEvaluator{
+
+    public LessThanOrEqualTo(RecordPointer record, FunctionArguments args) {
+      super(record, args);
+    }
+
+    @Override
+    public boolean valid(int i) {
+      return i <= 0;
+    }
+    
+  }
+  
+
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java
new file mode 100644
index 0000000..7b8902c
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java
@@ -0,0 +1,87 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn;
+
+import java.util.List;
+
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.exceptions.SetupException;
+
+import com.carrotsearch.hppc.ObjectIntOpenHashMap;
+
+public class FunctionArguments {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionArguments.class);
+  
+  private final boolean onlyConstants;
+  private final boolean includesAggregates;
+  private final List<BasicEvaluator> ev;  
+  private ObjectIntOpenHashMap<String> nameToPosition = new ObjectIntOpenHashMap<String>();
+  
+  public FunctionArguments(boolean onlyConstants, boolean includesAggregates, List<BasicEvaluator> evaluators, FunctionCall call){
+    this.onlyConstants = onlyConstants;
+    this.includesAggregates = includesAggregates;
+    String[] names = call.getDefinition().getArgumentNames();
+    for(int i =0; i < names.length; i++){
+      if(names[i] != null){
+        nameToPosition.put(names[i], i);
+      }
+    }
+    ev = evaluators;
+  }
+  
+  
+  public boolean includesAggregates() {
+    return includesAggregates;
+  }
+
+
+  public boolean isOnlyConstants(){
+    return onlyConstants;
+  }
+  
+  public BasicEvaluator getEvaluator(String name){
+    if(!nameToPosition.containsKey(name)) throw new RuntimeException("Unknown Item provided.");
+    int i = nameToPosition.lget();
+    return getEvaluator(i);
+  }
+  
+  public BasicEvaluator getEvaluator(int argIndex){
+    BasicEvaluator eval = ev.get(argIndex);
+    if(eval == null) throw new RuntimeException("Unknown Item provided.");
+    return eval;
+    
+  }
+  
+  /**
+   * Return this single argument evaluator in this set of evaluators.  If there isn't exactly one, throw a SetupException
+   * @return The single evaluator.
+   */
+  public BasicEvaluator getOnlyEvaluator() throws SetupException{
+    if(ev.size() != 1) throw new SetupException(String.format("Looking for a single argument.  Received %d arguments.", ev.size()));
+    return ev.get(0);
+  }
+  
+  public int size(){
+    return ev.size();
+  }
+  
+  public BasicEvaluator[] getArgsAsArray(){
+    return ev.toArray(new BasicEvaluator[size()]);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java
new file mode 100644
index 0000000..bc29161
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java
@@ -0,0 +1,26 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface FunctionEvaluator {
+  String value();
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java
new file mode 100644
index 0000000..719115a
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java
@@ -0,0 +1,110 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.exceptions.SetupException;
+import org.reflections.Reflections;
+import org.reflections.scanners.ResourcesScanner;
+import org.reflections.scanners.SubTypesScanner;
+import org.reflections.scanners.TypeAnnotationsScanner;
+import org.reflections.util.ClasspathHelper;
+import org.reflections.util.ConfigurationBuilder;
+import org.reflections.util.FilterBuilder;
+
+public class FunctionEvaluatorRegistry {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionEvaluatorRegistry.class);
+
+  static Map<String, Constructor<? extends BasicEvaluator>> map;
+
+  static {
+    final String scanPackage = "org.apache.drill.exec";
+
+    String s = FilterBuilder.Include.prefix(scanPackage);
+
+    Reflections r = new Reflections(new ConfigurationBuilder().filterInputsBy(new FilterBuilder().include(s))
+        .setUrls(ClasspathHelper.forPackage(scanPackage))
+        .setScanners(new SubTypesScanner(), new TypeAnnotationsScanner(), new ResourcesScanner()));
+
+    Set<Class<? extends BasicEvaluator>> providerClasses = r.getSubTypesOf(BasicEvaluator.class);
+    Map<String, Constructor<? extends BasicEvaluator>> funcs = new HashMap<String, Constructor<? extends BasicEvaluator>>();
+    for (Class<? extends BasicEvaluator> c : providerClasses) {
+      try {
+        FunctionEvaluator annotation = c.getAnnotation(FunctionEvaluator.class);
+        if (annotation == null) {
+          // only basic evaluator marked with a function evaluator interface will be examinged. 
+          continue;
+        }
+
+        Constructor<? extends BasicEvaluator> constructor = c.getConstructor(RecordPointer.class, FunctionArguments.class);
+//        if (!constructor.isAccessible()) {
+//          logger.warn("Unable to register Basic Evaluator {} because is not acccessible.", c);
+//          continue;
+//        }
+
+        funcs.put(annotation.value(), constructor);
+      } catch (NoSuchMethodException e) {
+        logger.warn(
+                "Unable to register Basic Evaluator {} because it does not have a constructor that accepts arguments of [SimpleRecord, ArgumentEvaluators] as its arguments.",
+                c);
+      } catch (SecurityException e) {
+        logger.warn("Unable to register Basic Evaluator {} because of security exception.", c, e);
+      }
+    }
+
+    map = Collections.unmodifiableMap(funcs);
+  }
+
+  public static BasicEvaluator getEvaluator(String name, FunctionArguments args, RecordPointer record) {
+    Constructor<? extends BasicEvaluator> c = map.get(name);
+    if (c == null) throw new SetupException(String.format("Unable to find requested basic evaluator %s.", name));
+    try {
+      try {
+        BasicEvaluator e = c.newInstance(record, args);
+        return e;
+      } catch (InvocationTargetException e) {
+        Throwable ex = e.getCause();
+        if (ex instanceof SetupException) {
+          throw (SetupException) ex;
+        } else {
+          if(ex instanceof RuntimeException){
+            throw (RuntimeException) ex;
+          }else{
+            throw new SetupException(String.format("Failure while attempting to create a new evaluator of type '%s'.", name));
+          }
+        }
+      }
+    } catch (RuntimeException | IllegalAccessException | InstantiationException ex) {
+      throw new SetupException(String.format("Failure while attempting to create a new evaluator of type '%s'.", name),
+          ex);
+    }
+
+  }
+
+  public static void main(String[] args) {
+    System.out.println("loaded.");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java
new file mode 100644
index 0000000..d080cd3
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java
@@ -0,0 +1,106 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.BaseBasicEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.exceptions.RecordException;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.NumericValue;
+import org.apache.drill.exec.ref.values.NumericValue.NumericType;
+import org.apache.drill.exec.ref.values.ScalarValues;
+
+public class MathEvaluators {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathEvaluators.class);
+  
+  @FunctionEvaluator("add")
+  public static class AddEvaluator extends BaseBasicEvaluator{
+    private final BasicEvaluator args[];
+
+    public AddEvaluator(RecordPointer record, FunctionArguments args){
+      super(args.isOnlyConstants(), record);
+      this.args = args.getArgsAsArray();
+    }
+    
+    @Override
+    public NumericValue eval() {
+      NumericValue[] values = new NumericValue[args.length];
+      for(int i =0; i < values.length; i++){
+        DataValue v = args[i].eval();
+        if(Types.isNumericType(v.getDataType())){
+          values[i] = v.getAsNumeric();
+        }
+      }
+      return NumericValue.add(values);
+    }
+
+    
+  }
+  
+  @FunctionEvaluator("multiply")
+  public static class MultiplyE extends BaseBasicEvaluator{
+    private final BasicEvaluator args[];
+
+    public MultiplyE(RecordPointer record, FunctionArguments args){
+      super(args.isOnlyConstants(), record);
+      this.args = args.getArgsAsArray();
+    }
+    
+    @Override
+    public NumericValue eval() {
+      long l = 1;
+      double d = 1;
+      boolean isFloating = false;
+      
+      for(int i =0; i < args.length; i++){
+        final DataValue v = args[i].eval();
+//        logger.debug("DataValue {}", v);
+        if(Types.isNumericType(v.getDataType())){
+          NumericValue n = v.getAsNumeric();
+          NumericType nt = n.getNumericType();
+//          logger.debug("Numeric Type: {}", nt);
+          if(isFloating || nt == NumericType.FLOAT || nt == NumericType.DOUBLE){
+            if(!isFloating){
+              d = l;
+              isFloating = true;
+            }
+            d *= n.getAsDouble();
+          }else{
+            l *= n.getAsLong();
+          }
+          
+        }else{
+          throw new RecordException(String.format("Unable to multiply a value of  %s.", v), null);
+        }
+      }
+      
+      NumericValue out = null;
+      if(isFloating){
+        out = new ScalarValues.DoubleScalar(d);
+      }else{
+        out = new ScalarValues.LongScalar(l);
+      }
+      
+      return out;
+    }
+
+    
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java
new file mode 100644
index 0000000..a3c336e
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java
@@ -0,0 +1,52 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.BaseBasicEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.values.ValueReader;
+import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
+
+public class StringEvaluators {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StringEvaluators.class);
+
+  @FunctionEvaluator("regex_like")
+  public static class RegexEvaluator extends BaseBasicEvaluator{
+    private final Matcher matcher;
+    private final BasicEvaluator eval;
+    
+    public RegexEvaluator(RecordPointer record, FunctionArguments args){
+      super(args.isOnlyConstants(), record);
+      matcher = Pattern.compile(ValueReader.getString(args.getEvaluator("pattern").eval())).matcher("");
+      eval = args.getEvaluator("value");
+    }
+    
+    @Override
+    public BooleanScalar eval() {
+      matcher.reset(ValueReader.getChars(eval.eval()));
+      return new BooleanScalar(matcher.find());
+    }
+  }
+  
+  
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java
new file mode 100644
index 0000000..ab7358f
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java
@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn.agg;
+
+import java.util.List;
+
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.values.DataValue;
+
+public class AggregatingWrapperEvaluator implements AggregatingEvaluator{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AggregatingWrapperEvaluator.class);
+
+  private List<AggregatingEvaluator> aggs;
+  private BasicEvaluator topEvaluator;
+  
+  public AggregatingWrapperEvaluator(List<AggregatingEvaluator> aggs, BasicEvaluator topEvaluator) {
+    super();
+    this.aggs = aggs;
+    this.topEvaluator = topEvaluator;
+  }
+
+  @Override
+  public DataValue eval() {
+    return topEvaluator.eval();
+  }
+
+  @Override
+  public boolean isConstant() {
+    return topEvaluator.isConstant();
+  }
+
+  @Override
+  public void addRecord() {
+    for(AggregatingEvaluator a : aggs){
+      a.addRecord();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java
new file mode 100644
index 0000000..099c001
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java
@@ -0,0 +1,75 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn.agg;
+
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.eval.fn.FunctionArguments;
+import org.apache.drill.exec.ref.eval.fn.FunctionEvaluator;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.ScalarValues;
+
+@FunctionEvaluator("count")
+public class CountAggregator implements AggregatingEvaluator{
+  
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CountAggregator.class);
+  
+  boolean isConstant;
+  long l = 0;
+  boolean checkForNull = false;
+  BasicEvaluator child;
+  
+  public CountAggregator(RecordPointer inputRecord, FunctionArguments e) {
+    isConstant = e.isOnlyConstants();
+    if(!e.getOnlyEvaluator().isConstant()){
+      checkForNull = true;
+      child = e.getOnlyEvaluator();
+    }
+  }
+  
+  
+  @Override
+  public void addRecord() {
+    if(checkForNull){
+      if(child.eval() != DataValue.NULL_VALUE){
+        l++;
+      }
+    }else{
+      l++;
+    }
+  }
+
+  @Override
+  public DataValue eval() {
+    DataValue v = new ScalarValues.LongScalar(l);
+    l = 0;
+    return v; 
+  }
+
+
+  @Override
+  public boolean isConstant() {
+    return isConstant;
+  }
+
+
+ 
+
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java
new file mode 100644
index 0000000..63fe017
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java
@@ -0,0 +1,103 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.eval.fn.agg;
+
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.eval.fn.FunctionArguments;
+import org.apache.drill.exec.ref.eval.fn.FunctionEvaluator;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.NumericValue;
+import org.apache.drill.exec.ref.values.ScalarValues;
+
+@FunctionEvaluator("sum")
+public class SumAggregator implements AggregatingEvaluator {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SumAggregator.class);
+  private boolean constantsOnly;
+  private long l = 0;
+  private double d = 0;
+
+  boolean integer = true;
+  private BasicEvaluator child;
+
+  public SumAggregator(RecordPointer inputRecord, FunctionArguments e) {
+    this.child = e.getOnlyEvaluator();
+    this.constantsOnly = e.isOnlyConstants();
+  }
+
+  @Override
+  public void addRecord() {
+    DataValue dv = child.eval();
+    NumericValue v = dv.getAsNumeric();
+    if (integer) {
+      
+      switch (v.getNumericType()) {
+      case DOUBLE:
+      case FLOAT:
+        integer = false;
+        d = l; // loss of precision
+        d += v.getAsDouble();
+        break;
+      case INT:
+      case LONG:
+        l += v.getAsLong();
+        return;
+      default:
+        throw new UnsupportedOperationException();
+      }
+    }else{
+      switch (v.getNumericType()) {
+      case DOUBLE:
+      case FLOAT:
+      case INT:
+      case LONG:
+        integer = false;
+        d += v.getAsDouble();
+        return;
+      default:
+        throw new UnsupportedOperationException();
+      }
+      
+    }
+  }
+
+  @Override
+  public DataValue eval() {
+    DataValue v;
+    if (integer) {
+      v = new ScalarValues.LongScalar(l);
+    } else {
+      v = new ScalarValues.DoubleScalar(d);
+    }
+    reset();
+    return v;
+  }
+
+  private void reset() {
+    l = 0;
+    d = 0;
+    integer = true;
+  }
+
+  @Override
+  public boolean isConstant() {
+    return constantsOnly;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java
new file mode 100644
index 0000000..643dc3f
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java
@@ -0,0 +1,46 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.exceptions;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class MajorException extends DrillRuntimeException{
+  @SuppressWarnings("unused") static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MajorException.class);
+
+  public MajorException() {
+    super();
+  }
+
+  public MajorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+
+  public MajorException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public MajorException(String message) {
+    super(message);
+  }
+
+  public MajorException(Throwable cause) {
+    super(cause);
+  }
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java
new file mode 100644
index 0000000..e4a272f
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java
@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.exceptions;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.ref.RecordPointer;
+
+public class RecordException extends DrillRuntimeException{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordException.class);
+  
+  private RecordPointer problemRecord;
+  
+  public RecordException(String message, RecordPointer problemRecord) {
+    super(message);
+    this.problemRecord = problemRecord;
+  }
+
+  public RecordException(String message, RecordPointer problemRecord, Throwable cause) {
+    super(message, cause);
+    this.problemRecord = problemRecord;
+  }
+
+  public RecordPointer getProblemRecord(){
+    return problemRecord;
+  }
+
+  public void setProblemRecord(RecordPointer problemRecord) {
+    this.problemRecord = problemRecord;
+  }
+  
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java
new file mode 100644
index 0000000..506b8bf
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java
@@ -0,0 +1,46 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.exceptions;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class SetupException extends DrillRuntimeException{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetupException.class);
+
+  public SetupException() {
+    super();
+  }
+
+  public SetupException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+
+  public SetupException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public SetupException(String message) {
+    super(message);
+  }
+
+  public SetupException(Throwable cause) {
+    super(cause);
+  }
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java
new file mode 100644
index 0000000..57966ce
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java
@@ -0,0 +1,66 @@
+package org.apache.drill.exec.ref.rops;
+
+import org.apache.drill.common.logical.data.SingleInputOperator;
+import org.apache.drill.exec.ref.RecordIterator;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.RecordIterator.NextOutcome;
+
+
+public abstract class AbstractBlockingOperator<T extends SingleInputOperator> extends SingleInputROPBase<T> {
+
+  public AbstractBlockingOperator(T config) {
+    super(config);
+  }
+
+  private RecordIterator incoming;
+  protected RecordPointer inputRecord;
+  protected final ProxySimpleRecord outputRecord = new ProxySimpleRecord();
+
+  @Override
+  public void setInput(RecordIterator incoming) {
+    this.incoming = incoming;
+    inputRecord = incoming.getRecordPointer();
+  }
+
+  protected abstract void consumeRecord();
+  protected abstract RecordIterator doWork();
+
+  private RecordIterator consumeData(){
+    while (incoming.next() != NextOutcome.NONE_LEFT)  {
+      consumeRecord();
+    }
+    return doWork();
+  }
+  
+  @Override
+  protected RecordIterator getIteratorInternal() {
+    return new BlockingIterator();
+  }
+
+  private class BlockingIterator implements RecordIterator{
+    private RecordIterator iter;
+    
+    public BlockingIterator(){
+    }
+    
+    @Override
+    public NextOutcome next() {
+      if(this.iter == null){
+        this.iter = consumeData();
+      }
+      return this.iter.next();
+    }
+    
+    @Override
+    public ROP getParent() {
+      return AbstractBlockingOperator.this;
+    }
+    
+    @Override
+    public RecordPointer getRecordPointer() {
+      return outputRecord;
+    }
+    
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java
new file mode 100644
index 0000000..d717fa3
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java
@@ -0,0 +1,113 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import java.io.IOException;
+import java.util.concurrent.locks.ReadWriteLock;
+
+import org.apache.drill.common.logical.data.SinkOperator;
+import org.apache.drill.exec.ref.RunOutcome;
+import org.apache.drill.exec.ref.RecordIterator;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
+import org.apache.drill.exec.ref.RecordIterator.NextOutcome;
+import org.apache.drill.exec.ref.eval.EvaluatorFactory;
+import org.apache.drill.exec.ref.exceptions.SetupException;
+
+public abstract class BaseSinkROP<T extends SinkOperator> extends SingleInputROPBase<T> implements SinkROP{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseSinkROP.class);
+  
+  protected RecordIterator iter;
+  protected RecordPointer record;
+  
+  public BaseSinkROP(T config) {
+    super(config);
+  }
+  
+  @Override
+  protected void setupEvals(EvaluatorFactory builder) throws SetupException {
+    try {
+      setupSink();
+    } catch (IOException e) {
+      throw new SetupException(String.format("failure setting up %s sink rop.", this.getClass()), e);
+    }
+  }
+  
+  @Override
+  protected void setInput(RecordIterator incoming) {
+    iter = incoming;
+    record = incoming.getRecordPointer();
+  }
+
+  @Override
+  public RecordIterator getIteratorInternal() {
+    throw new UnsupportedOperationException("A ReferenceSink");
+  }
+  
+  @Override
+  public RunOutcome run(StatusHandle handle) {
+    Throwable exception = null;
+    final int runsize = 1000;
+    int recordCount = 0;
+    OutcomeType outcome = OutcomeType.FAILED;
+    long pos = -1; 
+    try{
+    while(true){
+      boolean more = true;
+      for(;recordCount < runsize; recordCount++){
+        NextOutcome r = iter.next();
+        if(r == NextOutcome.NONE_LEFT){
+          more = false;
+          break;
+        }else{
+          pos = sinkRecord(record);
+        }
+      }
+      handle.progress(pos, recordCount);
+      if(!handle.okToContinue()){
+        logger.debug("Told to cancel, breaking run.");
+        outcome = OutcomeType.CANCELED;
+        break;
+      }else if(!more){
+        outcome = OutcomeType.SUCCESS;
+        logger.debug("Breaking because no more records were found.");
+        break;
+      }else{
+        logger.debug("No problems, doing next progress iteration.");
+      }
+      
+    }
+    }catch(Exception e){
+      exception = e ;
+    }
+    
+    cleanup(outcome);
+    return new RunOutcome(outcome, pos, recordCount, exception);
+    
+  }
+
+  /**
+   * 
+   * @param r RecordPointer to record
+   * @return The approximate amount of bytes written.
+   * @throws IOException
+   */
+  public abstract long sinkRecord(RecordPointer r) throws IOException;
+  protected abstract void setupSink() throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java
new file mode 100644
index 0000000..535281c
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java
@@ -0,0 +1,25 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import org.apache.drill.common.expression.SchemaPath;
+
+public interface BoundaryListener {
+  public SchemaPath[] getOrderedBoundaryPaths();
+  public void markBoundaryAsCrossed();
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java
new file mode 100644
index 0000000..8b9c2f6
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java
@@ -0,0 +1,269 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.data.CollapsingAggregate;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.ref.RecordIterator;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.UnbackedRecord;
+import org.apache.drill.exec.ref.eval.EvaluatorFactory;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.ScalarValues;
+
+public class CollapsingAggregateROP extends SingleInputROPBase<CollapsingAggregate> implements BoundaryListener {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CollapsingAggregateROP.class);
+
+  private SchemaPath[] boundaryPaths;
+  private BasicEvaluator targetEvaluator;
+  private boolean foundTarget = false;
+  private DataValue currentValue;
+  private DataValue previousValue;
+  private BasicEvaluator boundaryKey;
+  private RecordIterator incoming;
+  private BasicEvaluator[] carryovers;
+  private FieldReference[] carryoverNames;
+  private DataValue[] carryoverValues;
+  private AggregatingEvaluator[] aggs;
+  private SchemaPath[] aggNames;
+  private boolean boundaryCrossed = false;
+  private final boolean targetMode;
+  
+  public CollapsingAggregateROP(CollapsingAggregate config) {
+    super(config);
+    targetMode = config.getTarget() != null;
+  }
+  
+  @Override
+  protected void setupEvals(EvaluatorFactory builder) {
+    
+    if(config.getWithin() != null){
+      boundaryPaths = new SchemaPath[]{config.getWithin()};
+      boundaryKey = builder.getBasicEvaluator(record, config.getWithin());  
+    }else{
+      boundaryPaths = new SchemaPath[0];
+      boundaryKey = new ScalarValues.IntegerScalar(0);
+    }
+    
+    aggs = new AggregatingEvaluator[config.getAggregations().length];
+    carryovers = new BasicEvaluator[config.getCarryovers().length];
+    carryoverNames = new FieldReference[config.getCarryovers().length];
+    carryoverValues = new DataValue[config.getCarryovers().length];
+
+    if(targetMode){
+      targetEvaluator = builder.getBasicEvaluator(record, config.getTarget());
+    }
+    aggNames = new SchemaPath[aggs.length];
+    for(int i =0; i < aggs.length; i++){
+      aggs[i] = builder.getAggregatingOperator(record, config.getAggregations()[i].getExpr());
+      aggNames[i] = config.getAggregations()[i].getRef();
+    }
+    
+    for(int i =0; i < carryovers.length; i++){
+      carryovers[i] = builder.getBasicEvaluator(record, config.getCarryovers()[i]);
+      carryoverNames[i] = config.getCarryovers()[i];
+    }
+    
+    
+    
+  }
+ 
+  @Override
+  protected void setInput(RecordIterator incoming) {
+    this.incoming = incoming;
+  }
+
+  @Override
+  protected RecordIterator getIteratorInternal() {
+    return new AggregatingIterator();
+  }
+
+  private boolean checkBoundaryCrossing() {
+    if (boundaryCrossed) {
+      boundaryCrossed = false;
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  @Override
+  public void markBoundaryAsCrossed() {
+    this.boundaryCrossed = true;
+  }
+
+  
+
+  
+  private class AggregatingIterator implements RecordIterator {
+    private boolean remainder = false;
+    private boolean more = true;
+    private UnbackedRecord outputRecord = new UnbackedRecord();
+    public AggregatingIterator() {
+    }
+
+    private NextOutcome readNext(){
+
+      // copy over the previous values.
+      previousValue = currentValue;
+      
+      // increment the parent forward.
+      NextOutcome n = incoming.next();
+
+      boolean changed = false;
+      
+      // read boundary values unless no new values were loaded.
+      if(n != NextOutcome.NONE_LEFT){
+        currentValue = boundaryKey.eval();
+        if(!currentValue.equals(previousValue)) changed = true;
+  
+        // skip first boundary.
+        if(previousValue == null) changed = false;
+        
+      }else{
+        
+        changed = true;
+      }
+      
+      if(changed){
+        markBoundaryAsCrossed();
+      }
+      
+      return n;
+    }
+    
+    private void consumeCurrent(){
+      for(int x = 0; x < aggs.length; x++){
+        aggs[x].addRecord();  
+      }
+      
+      // if we're in target mode and this row matches the target criteria, we're going to copy carry over values and mark foundTarget = true.
+      if(targetMode){
+        DataValue v = targetEvaluator.eval();
+        if(v.getDataType().getMinorType() == MinorType.BIT && v.getAsBooleanValue().getBoolean()){
+          foundTarget = true;
+          for(int i =0 ; i < carryovers.length; i++){
+            carryoverValues[i] = carryovers[i].eval();
+          }
+        }
+      }else{
+        for(int i =0 ; i < carryovers.length; i++){
+          carryoverValues[i] = carryovers[i].eval();
+        }
+      }
+    }
+    
+    /** 
+     * Write the output of the operation to the unbacked record.
+     * @return Whether or not the record should be included in the output.
+     */
+    private boolean writeOutputRecord(){
+      outputRecord.clear();
+      for(int x = 0; x < aggs.length; x++){
+        DataValue dv = aggs[x].eval();
+//        logger.debug("Adding Aggregated Values named {} with value {}", outputNames[x], dv);
+        outputRecord.addField(aggNames[x], dv);
+      }
+      
+      // Add the carryover keys.
+      if(targetMode){
+        if(foundTarget){
+          for(int y = 0; y < carryoverNames.length; y++){
+            outputRecord.addField(carryoverNames[y], carryoverValues[y]);
+          }
+          foundTarget = false;
+          return true;
+        }else{
+          return false;
+        }
+      }else{
+        for(int y = 0; y < carryoverNames.length; y++){
+          outputRecord.addField(carryoverNames[y], carryoverValues[y]);
+        }
+        return true;
+      }
+      
+
+      
+      
+    }
+    
+    @Override
+    public NextOutcome next() {
+      outside: while(true){
+      NextOutcome whatNext = null;
+      
+      // if we don't have more and there are no more values, exit.
+      if (!more) return NextOutcome.NONE_LEFT;
+      
+      while (true) {
+        
+        // we shouldn't increment the iterator since we have to consume our remainder.
+        if(!remainder){
+          whatNext = readNext();  
+
+          // if we've just crossed a boundary, we should output the previous values.
+          if(checkBoundaryCrossing()){
+            boolean wroteSomething = writeOutputRecord();
+            
+            // if there is no future input, we'll flag as !more.  Otherwise, will inform that there are pending records that should be consumed.
+            if(whatNext == NextOutcome.NONE_LEFT){
+              more = false;
+            }else{
+              remainder = true;
+            }
+            
+            // if we didn't write something, we'll retry the outside loop
+            if(!wroteSomething) continue outside;
+            
+            // always return a next outcome of true since we've just output a record.
+            return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
+            
+          }
+        }
+        
+        // we don't consume the current record until after we've output a record associated with a boundary (as necessary).
+        consumeCurrent();
+        remainder = false;
+      }
+      
+      }
+    }
+
+    @Override
+    public ROP getParent() {
+      return CollapsingAggregateROP.this;
+    }
+
+    @Override
+    public RecordPointer getRecordPointer() {
+      return outputRecord;
+    }
+
+  }
+
+  @Override
+  public SchemaPath[] getOrderedBoundaryPaths() {
+    return boundaryPaths;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java
new file mode 100644
index 0000000..5d6f91f
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java
@@ -0,0 +1,109 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.logical.data.Constant;
+import org.apache.drill.exec.ref.IteratorRegistry;
+import org.apache.drill.exec.ref.RecordIterator;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
+import org.apache.drill.exec.ref.UnbackedRecord;
+import org.apache.drill.exec.ref.exceptions.SetupException;
+import org.apache.drill.exec.ref.rse.JSONRecordReader;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class ConstantROP extends ROPBase<Constant>{
+    static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanROP.class);
+
+    private UnbackedRecord record;
+
+    public ConstantROP(Constant config) {
+        super(config);
+        record = new UnbackedRecord();
+    }
+
+
+    @Override
+    protected void setupIterators(IteratorRegistry registry) throws SetupException {
+       // try{
+            super.setupIterators(registry);
+            // need to assign reader
+           // throw new IOException();
+        //}catch(IOException e){
+            //throw new SetupException("Failure while setting up reader.");
+        //}
+    }
+
+
+    @Override
+    protected RecordIterator getIteratorInternal() {
+        return new ConstantIterator(ConstantROP.this.config.getContent().getRoot());
+    }
+
+
+    @Override
+    public void cleanup(OutcomeType outcome) {
+        super.cleanup(outcome);
+    }
+
+
+    class ConstantIterator implements RecordIterator {
+
+        Iterator<JsonNode> jsonIter;
+
+        ConstantIterator(JsonNode json) {
+            jsonIter = json.elements();
+        }
+
+        public RecordPointer getRecordPointer(){
+            return record;
+        }
+
+        public NextOutcome next(){
+            if ( ! jsonIter.hasNext()){
+                return NextOutcome.NONE_LEFT;
+            }
+            JsonNode contentJSON = ConstantROP.this.config.getContent().getRoot();
+            if (contentJSON.isArray())
+            { // list of constant records was specified
+                JsonNode node;
+                node = jsonIter.next();
+                convertJsonToRP(node, record);
+                return NextOutcome.INCREMENTED_SCHEMA_UNCHANGED;
+            }
+            else{
+                convertJsonToRP(contentJSON, record);
+                return NextOutcome.NONE_LEFT;
+            }
+        }
+
+        private void convertJsonToRP(JsonNode node, RecordPointer rp){
+            record.clear();
+            record.merge(JSONRecordReader.convert(node));
+        }
+
+        public ROP getParent(){
+            return ConstantROP.this;
+        }
+
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java
new file mode 100644
index 0000000..8481daa
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java
@@ -0,0 +1,49 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import java.io.IOException;
+
+public interface DataWriter {
+  public void startRecord() throws IOException;
+  public void writeArrayStart(int length) throws IOException;
+  public void writeArrayElementStart() throws IOException;
+  public void writeArrayElementEnd() throws IOException;
+  public void writeArrayEnd() throws IOException;
+  
+  public void writeMapStart() throws IOException;
+  public void writeMapKey(CharSequence seq) throws IOException;
+  public void writeMapValueStart() throws IOException;
+  public void writeMapValueEnd() throws IOException;
+  public void writeMapEnd() throws IOException;
+  
+  public void writeBoolean(boolean b) throws IOException;
+  public void writeSInt32(int value) throws IOException;
+  public void writeSInt64(long value) throws IOException;
+  public void writeBytes(byte[] bytes) throws IOException;
+  public void writeSFloat64(double value) throws IOException;
+  public void writeSFloat32(float value) throws IOException;
+  public void writeNullValue() throws IOException;
+  public void writeCharSequence(CharSequence value) throws IOException;
+  public void endRecord() throws IOException;
+  public void finish() throws IOException;
+  
+  public enum ConverterType{
+    JSON
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java
new file mode 100644
index 0000000..e87c3fb
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java
@@ -0,0 +1,81 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.exec.ref.RecordIterator;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.eval.EvaluatorFactory;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BooleanEvaluator;
+
+public class FilterROP extends SingleInputROPBase<Filter>{
+
+  private FilterIterator iter;
+  private BooleanEvaluator filterEval;
+  
+  public FilterROP(Filter config) {
+    super(config);
+  }
+
+  @Override
+  protected void setupEvals(EvaluatorFactory builder) {
+    filterEval = builder.getBooleanEvaluator(record, config.getExpr());
+  }
+
+  @Override
+  public void setInput(RecordIterator incoming) {
+    iter = new FilterIterator(incoming);
+  }
+
+  @Override
+  public RecordIterator getIteratorInternal() {
+    return iter;
+  }
+  
+  private class FilterIterator implements RecordIterator{
+    RecordIterator incoming;
+
+    public FilterIterator(RecordIterator incoming) {
+      this.incoming = incoming;
+    }
+
+    @Override
+    public NextOutcome next() {
+      NextOutcome r;
+      while(true){
+        r = incoming.next();
+        if(r == NextOutcome.NONE_LEFT) return NextOutcome.NONE_LEFT;
+        if(filterEval.eval()) return r;
+      }
+    }
+
+    @Override
+    public ROP getParent() {
+      return FilterROP.this;
+    }
+
+    @Override
+    public RecordPointer getRecordPointer() {
+      return record;
+    }
+    
+    
+    
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a593d908/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java
new file mode 100644
index 0000000..f6e6907
--- /dev/null
+++ b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java
@@ -0,0 +1,140 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ref.rops;
+
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.data.Flatten;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.exec.ref.RecordIterator;
+import org.apache.drill.exec.ref.RecordPointer;
+import org.apache.drill.exec.ref.UnbackedRecord;
+import org.apache.drill.exec.ref.eval.EvaluatorFactory;
+import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
+import org.apache.drill.exec.ref.values.BaseArrayValue;
+import org.apache.drill.exec.ref.values.DataValue;
+import org.apache.drill.exec.ref.values.SimpleArrayValue;
+
+
+public class FlattenROP extends SingleInputROPBase<Flatten> {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenROP.class);
+
+  private RecordPointer outputRecord = new UnbackedRecord();
+  private BasicEvaluator evaluator;
+  private RecordIterator iter;
+
+  public FlattenROP(Flatten config) {
+    super(config);
+  }
+
+  @Override
+  protected void setInput(RecordIterator incoming) {
+    this.iter = new FlattenIterator(incoming);
+  }
+
+  @Override
+  protected RecordIterator getIteratorInternal() {
+    return iter;
+  }
+
+  @Override
+  protected void setupEvals(EvaluatorFactory builder) {
+    evaluator = builder.getBasicEvaluator(record, config.getExpr());
+  }
+
+  private class ArrayValueIterator {
+    private BaseArrayValue arrayValue;
+    private int currentIndex = 0;
+
+    public ArrayValueIterator(BaseArrayValue arrayValue) {
+      this.arrayValue = arrayValue;
+    }
+
+    public ArrayValueIterator() {
+      this(new SimpleArrayValue());
+    }
+
+    public DataValue next() {
+      DataValue v = null;
+      if (currentIndex < arrayValue.size()) {
+        v = arrayValue.getByArrayIndex(currentIndex);
+      }
+
+      currentIndex++;
+      return v;
+
+    }
+  }
+
+  private class FlattenIterator implements RecordIterator {
+    RecordIterator incoming;
+    NextOutcome currentOutcome;
+    int currentIndex = 0;
+    ArrayValueIterator arrayValueIterator = new ArrayValueIterator();
+
+    public FlattenIterator(RecordIterator incoming) {
+      super();
+      this.incoming = incoming;
+    }
+
+    @Override
+    public RecordPointer getRecordPointer() {
+      return outputRecord;
+    }
+
+    @Override
+    public NextOutcome next() {
+      DataValue v;
+      if ((v = arrayValueIterator.next()) != null)  //if we are already iterating through a sub-array, keep going
+        return mergeValue(v);
+      else //otherwise, get the next record
+        currentOutcome = incoming.next();
+
+
+      if (currentOutcome != NextOutcome.NONE_LEFT) {
+        if (evaluator.eval().getDataType().getMode() == DataMode.REPEATED) {
+          arrayValueIterator = new ArrayValueIterator(evaluator.eval().getAsContainer().getAsArray());
+
+          while ((v = arrayValueIterator.next()) != null) {
+            return mergeValue(v);
+          }
+        } else {
+          outputRecord.copyFrom(record);
+          outputRecord.addField(config.getName(), evaluator.eval());
+          if(config.isDrop())
+            outputRecord.removeField((SchemaPath)config.getExpr());
+        }
+      }
+      return currentOutcome;
+    }
+
+    // helper function to merge one of the values from a sub array into the parent record
+    private NextOutcome mergeValue(DataValue v) {
+      outputRecord.copyFrom(record);
+      outputRecord.addField(config.getName(), v);
+      if(config.isDrop())
+        outputRecord.removeField((SchemaPath)config.getExpr());
+      return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
+    }
+
+    @Override
+    public ROP getParent() {
+      return FlattenROP.this;
+    }
+
+  }
+}