You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/02 05:03:05 UTC
[05/13] git commit: - Implement support for VARCHAR in
EvaluationVisitor - Remove ByteHolder - Add toString() helper for VarLen
value holders - Add MinorType accessor to code generators HoldingContainer -
Fix BitVector.generateTestData() - Implemen
- Implement support for VARCHAR in EvaluationVisitor
- Remove ByteHolder
- Add toString() helper for VarLen value holders
- Add MinorType accessor to code generators HoldingContainer
- Fix BitVector.generateTestData()
- Implement draft of SUBSTRING function
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/33991841
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/33991841
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/33991841
Branch: refs/heads/master
Commit: 3399184121a7dac15a316ee9ae529c6ddf90c5f5
Parents: 128a09d
Author: Ben Becker <be...@gmail.com>
Authored: Thu Aug 1 17:55:30 2013 -0700
Committer: Ben Becker <be...@gmail.com>
Committed: Thu Aug 1 17:55:30 2013 -0700
----------------------------------------------------------------------
.../ValueVectors/templates/ValueHolders.java | 12 ++++
.../templates/VariableLengthVectors.java | 9 ---
.../apache/drill/exec/expr/CodeGenerator.java | 5 ++
.../drill/exec/expr/EvaluationVisitor.java | 29 ++++++---
.../drill/exec/expr/fn/impl/Substring.java | 68 ++++++++++++++++++++
.../exec/physical/config/MockRecordReader.java | 3 +-
.../physical/impl/filter/FilterTemplate.java | 2 +-
.../org/apache/drill/exec/vector/BitVector.java | 2 +-
.../apache/drill/exec/vector/ByteHolder.java | 12 ----
.../exec/physical/impl/TestSimpleFunctions.java | 43 +++++++++++++
.../test/resources/functions/testSubstring.json | 37 +++++++++++
11 files changed, 187 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
index dad7712..18be0f3 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
@@ -43,7 +43,19 @@ public final class ${className} implements ValueHolder{
/** The buffer holding actual values. **/
public ByteBuf buffer;
+
+ public String toString() {
+ <#if mode.name == "Optional">
+ if (isSet == 0)
+ return "<NULL>";
</#if>
+ byte[] buf = new byte[end-start];
+ buffer.getBytes(start, buf, 0, end-start);
+ return new String(buf);
+ }
+
+ </#if>
+
<#else>
/** The first index (inclusive) into the Vector. **/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
index 0871b82..40363a7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
@@ -20,7 +20,6 @@ import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.ByteHolder;
import org.mortbay.jetty.servlet.Holder;
import com.google.common.base.Charsets;
@@ -186,14 +185,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
holder.end = offsetVector.getAccessor().get(index + 1);
holder.buffer = data;
}
-
- public void get(int index, ByteHolder holder){
- assert index >= 0;
- holder.start = offsetVector.getAccessor().get(index);
- holder.length = offsetVector.getAccessor().get(index + 1) - holder.start;
- assert holder.length >= 0;
- holder.buffer = offsetVector.data;
- }
public Object getObject(int index) {
return get(index);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
index 13c7781..67ca614 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.compile.TemplateClassDefinition;
@@ -190,6 +191,10 @@ public class CodeGenerator<T> {
public boolean isRepeated(){
return type.getMode() == DataMode.REPEATED;
}
+
+ public TypeProtos.MinorType getMinorType() {
+ return type.getMinorType();
+ }
}
public JType getHolderType(MajorType t){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 61f6ddc..fc1068b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -10,6 +10,7 @@ import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.FunctionHolder;
@@ -157,8 +158,11 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
JConditional jc = block._if(hc.getIsSet().eq(JExpr.lit(0)).not());
block = jc._then();
}
- block.add(vv.invoke("getMutator").invoke("set").arg(JExpr.direct("outIndex")).arg(hc.getValue()));
-
+ if (hc.getMinorType() == TypeProtos.MinorType.VARCHAR) {
+ block.add(vv.invoke("getMutator").invoke("set").arg(JExpr.direct("outIndex")).arg(hc.getHolder()));
+ } else {
+ block.add(vv.invoke("getMutator").invoke("set").arg(JExpr.direct("outIndex")).arg(hc.getValue()));
+ }
return null;
}
@@ -173,7 +177,7 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
JVar obj = generator.getSetupBlock().decl( //
generator.getModel()._ref(Object.class), //
generator.getNextVar("obj"), //
- JExpr.direct("outgoing").invoke("getValueVectorById").arg(JExpr.lit(e.getFieldId())).arg( ((JClass)vvType).dotclass()));
+ JExpr.direct("incoming").invoke("getValueVectorById").arg(JExpr.lit(e.getFieldId())).arg( ((JClass)vvType).dotclass()));
generator.getSetupBlock().assign(vv1, JExpr.cast(vvType, obj));
// evaluation work.
@@ -184,14 +188,19 @@ public class EvaluationVisitor extends AbstractExprVisitor<HoldingContainer, Cod
JBlock blk = generator.getBlock();
blk.assign(out.getIsSet(), vv1.invoke("getAccessor").invoke("isSet").arg(JExpr.direct("inIndex")));
JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1)));
- jc._then() //
- .assign(out.getValue(), vv1.invoke("getAccessor").invoke("get").arg(JExpr.direct("inIndex"))); //
- //.assign(out.getIsSet(), JExpr.lit(1));
- //jc._else()
- //.assign(out.getIsSet(), JExpr.lit(0));
-
+ if (e.getMajorType().getMinorType() == TypeProtos.MinorType.VARCHAR) {
+ jc._then()
+ .add(vv1.invoke("getAccessor").invoke("get").arg(JExpr.direct("inIndex")).arg(out.getHolder()));
+ } else {
+ jc._then()
+ .assign(out.getValue(), vv1.invoke("getAccessor").invoke("get").arg(JExpr.direct("inIndex")));
+ }
}else{
- generator.getBlock().assign(out.getValue(), vv1.invoke("getAccessor").invoke("get").arg(JExpr.direct("inIndex")));
+ if (e.getMajorType().getMinorType() == TypeProtos.MinorType.VARCHAR) {
+ generator.getBlock().add(vv1.invoke("getAccessor").invoke("get").arg(JExpr.direct("inIndex")).arg(out.getHolder()));
+ } else {
+ generator.getBlock().assign(out.getValue(), vv1.invoke("getAccessor").invoke("get").arg(JExpr.direct("inIndex")));
+ }
}
return out;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Substring.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Substring.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Substring.java
new file mode 100644
index 0000000..785ed7d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Substring.java
@@ -0,0 +1,68 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.common.expression.ArgumentValidators;
+import org.apache.drill.common.expression.CallProvider;
+import org.apache.drill.common.expression.FunctionDefinition;
+import org.apache.drill.common.expression.OutputTypeDeterminer;
+import org.apache.drill.exec.expr.DrillFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.*;
+
+// TODO: implement optional length parameter
+// implement negative start position
+
+@FunctionTemplate(name = "substring",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+public class Substring implements DrillFunc {
+
+ @Param VarCharHolder string;
+ @Param BigIntHolder offset;
+ @Param BigIntHolder length;
+ @Output VarCharHolder out;
+
+ @Override
+ public void setup(RecordBatch incoming) { }
+
+ @Override
+ public void eval() {
+ out.buffer = string.buffer;
+ out.start = (int)offset.value;
+ out.end = (int)length.value;
+ }
+
+ public static class Provider implements CallProvider {
+
+ @Override
+ public FunctionDefinition[] getFunctionDefintions() {
+ return new FunctionDefinition[] {
+ FunctionDefinition.simple("substring",
+ new ArgumentValidators.AnyTypeAllowed(3),
+ new OutputTypeDeterminer.SameAsFirstInput(),
+ "substring",
+ "substr")
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
index a477925..6e3a4aa 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
@@ -99,9 +99,8 @@ public class MockRecordReader implements RecordReader {
logger.debug("MockRecordReader: Generating random data for VV of type " + v.getClass().getName());
ValueVector.Mutator m = v.getMutator();
- m.generateTestData();
-
m.setValueCount(recordSetSize);
+ m.generateTestData();
}
return recordSetSize;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
index 92a6994..bf3cd91 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
@@ -40,7 +40,6 @@ public abstract class FilterTemplate implements Filterer{
}
public void filterBatch(int recordCount){
- doTransfers();
switch(svMode){
case NONE:
filterBatchNoSV(recordCount);
@@ -51,6 +50,7 @@ public abstract class FilterTemplate implements Filterer{
default:
throw new UnsupportedOperationException();
}
+ doTransfers();
}
private void filterBatchSV2(int recordCount){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index dceff1c..1376b7d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -191,7 +191,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
@Override
public final void generateTestData() {
boolean even = true;
- for (int i = 0; i < valueCount; i++, even = !even) {
+ for (int i = 0; i < valueCapacity; i++, even = !even) {
if (even) {
set(i, 1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java
deleted file mode 100644
index 45d8019..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.ByteBuf;
-
-public class ByteHolder {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteHolder.class);
-
- public ByteBuf buffer;
- public int start;
- public int length;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index d663401..bed976c 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -24,6 +24,8 @@ import com.yammer.metrics.MetricRegistry;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -35,6 +37,8 @@ import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.NullableVarCharHolder;
+import org.apache.drill.exec.vector.NullableVarCharVector;
import org.junit.After;
import org.junit.Test;
@@ -100,6 +104,45 @@ public class TestSimpleFunctions {
}
+ @Test
+ public void testSubstring(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Throwable{
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstring.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, ExecProtos.FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while(exec.next()){
+ NullableVarCharVector c1 = exec.getValueVectorById(new SchemaPath("col3", ExpressionPosition.UNKNOWN), NullableVarCharVector.class);
+ NullableVarCharVector.Accessor a1;
+ a1 = c1.getAccessor();
+
+ int count = 0;
+ for(int i = 0; i < c1.getAccessor().getValueCount(); i++){
+ if (!a1.isNull(i)) {
+ NullableVarCharHolder holder = new NullableVarCharHolder();
+ a1.get(i, holder);
+ assertEquals("aaaa", holder.toString());
+ ++count;
+ }
+ }
+ assertEquals(50, count);
+ }
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+ assertTrue(!context.isFailed());
+
+ }
+
@After
public void tearDown() throws Exception{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/33991841/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
new file mode 100644
index 0000000..33b0da0
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
@@ -0,0 +1,37 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "yellow", type: "VARCHAR", mode: "OPTIONAL"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"project",
+ exprs: [
+ { ref: "col3", expr:"substring(yellow, 0, 4)" }
+ ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file