You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/06/04 20:48:36 UTC
[1/3] drill git commit: DRILL-3155: Part 1: Move complex vectors
logic in the correct package.
Repository: drill
Updated Branches:
refs/heads/master 69b39c110 -> 287f52db0
DRILL-3155: Part 1: Move complex vectors logic in the correct package.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/21de1381
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/21de1381
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/21de1381
Branch: refs/heads/master
Commit: 21de138161a12ff0d01542950601b7944e9526ed
Parents: 69b39c1
Author: Mehant Baid <me...@gmail.com>
Authored: Fri May 29 14:47:23 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Wed Jun 3 15:07:25 2015 -0700
----------------------------------------------------------------------
.../codegen/templates/RepeatedValueVectors.java | 4 +-
.../impl/flatten/FlattenRecordBatch.java | 2 +-
.../physical/impl/flatten/FlattenTemplate.java | 2 +-
.../exec/physical/impl/flatten/Flattener.java | 2 +-
.../apache/drill/exec/store/VectorHolder.java | 107 ----------
.../columnreaders/FixedWidthRepeatedReader.java | 4 +-
.../columnreaders/ParquetRecordReader.java | 2 +-
.../drill/exec/vector/AllocationHelper.java | 3 +-
.../exec/vector/BaseRepeatedValueVector.java | 207 ------------------
.../drill/exec/vector/ContainerVectorLike.java | 39 ----
.../vector/RepeatedFixedWidthVectorLike.java | 56 -----
.../drill/exec/vector/RepeatedMutator.java | 22 --
.../drill/exec/vector/RepeatedValueVector.java | 82 -------
.../vector/RepeatedVariableWidthVectorLike.java | 47 ----
.../vector/complex/BaseRepeatedValueVector.java | 213 +++++++++++++++++++
.../vector/complex/ContainerVectorLike.java | 43 ++++
.../complex/RepeatedFixedWidthVectorLike.java | 56 +++++
.../exec/vector/complex/RepeatedListVector.java | 5 -
.../exec/vector/complex/RepeatedMapVector.java | 4 -
.../vector/complex/RepeatedValueVector.java | 86 ++++++++
.../RepeatedVariableWidthVectorLike.java | 47 ++++
21 files changed, 453 insertions(+), 580 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 7b2b78d..813c3f8 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -19,9 +19,7 @@
import java.lang.Override;
import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.BaseRepeatedValueVector;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike;
+import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector;
import org.mortbay.jetty.servlet.Holder;
<@pp.dropOutputFile />
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 00a78fd..9991404 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -50,7 +50,7 @@ import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.vector.RepeatedValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedValueVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.RepeatedMapVector;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
index b8d040c..de67b62 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
@@ -31,7 +31,7 @@ import org.apache.drill.exec.record.selection.SelectionVector4;
import com.google.common.collect.ImmutableList;
-import org.apache.drill.exec.vector.RepeatedValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedValueVector;
public abstract class FlattenTemplate implements Flattener {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenTemplate.class);
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
index 323bf43..92cf79d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
@@ -24,7 +24,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.RepeatedValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedValueVector;
public interface Flattener {
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
deleted file mode 100644
index e602fd7..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike;
-import org.apache.drill.exec.vector.RepeatedMutator;
-import org.apache.drill.exec.vector.RepeatedVariableWidthVectorLike;
-import org.apache.drill.exec.vector.ValueVector;
-
-public class VectorHolder {
- private int count;
- private int groupCount;
- private int length;
- private ValueVector vector;
- private int currentLength;
- private boolean repeated;
-
-
- public VectorHolder(int length, ValueVector vector) {
- this.length = length;
- this.vector = vector;
- if (vector instanceof RepeatedFixedWidthVectorLike || vector instanceof RepeatedVariableWidthVectorLike) {
- repeated = true;
- }
- }
-
- public VectorHolder(ValueVector vector) {
- this.length = vector.getValueCapacity();
- this.vector = vector;
- if (vector instanceof RepeatedFixedWidthVectorLike || vector instanceof RepeatedVariableWidthVectorLike) {
- repeated = true;
- }
- }
-
- public boolean isRepeated() {
- return repeated;
- }
-
- public ValueVector getValueVector() {
- return vector;
- }
-
- public void incAndCheckLength(int newLength) {
- if (!hasEnoughSpace(newLength)) {
- throw new BatchExceededException(length, vector.getBufferSize() + newLength);
- }
-
- currentLength += newLength;
- count += 1;
- }
-
- public void setGroupCount(int groupCount) {
- if (this.groupCount < groupCount) {
- RepeatedMutator mutator = (RepeatedMutator) vector.getMutator();
- while (this.groupCount < groupCount) {
- mutator.startNewGroup(++this.groupCount);
- }
- }
- }
-
- public boolean hasEnoughSpace(int newLength) {
- return length >= currentLength + newLength;
- }
-
- public int getLength() {
- return length;
- }
-
- public void reset() {
- currentLength = 0;
- count = 0;
- allocateNew(length);
- }
-
- public void populateVectorLength() {
- ValueVector.Mutator mutator = vector.getMutator();
- if (vector instanceof RepeatedFixedWidthVectorLike || vector instanceof RepeatedVariableWidthVectorLike) {
- mutator.setValueCount(groupCount);
- } else {
- mutator.setValueCount(count);
- }
- }
-
- public void allocateNew(int valueLength) {
- AllocationHelper.allocate(vector, valueLength, 10, 5);
- }
-
- public void allocateNew(int valueLength, int repeatedPerTop) {
- AllocationHelper.allocate(vector, valueLength, 10, repeatedPerTop);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java
index 2b929a4..d6e30ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java
@@ -21,10 +21,8 @@ import java.io.IOException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike;
-import org.apache.drill.exec.vector.RepeatedValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedValueVector;
import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
import parquet.format.SchemaElement;
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
index 0cbd480..da6fbfb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
@@ -41,7 +41,7 @@ import org.apache.drill.exec.store.AbstractRecordReader;
import org.apache.drill.exec.store.parquet.DirectCodecFactory;
import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.NullableIntVector;
-import org.apache.drill.exec.vector.RepeatedValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedValueVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
index eddefd0..e518042 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.vector;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike;
+import org.apache.drill.exec.vector.complex.RepeatedVariableWidthVectorLike;
public class AllocationHelper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
deleted file mode 100644
index d5a0d62..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import java.util.Collections;
-import java.util.Iterator;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.exception.SchemaChangeRuntimeException;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
-
-public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
-
- public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
- public final static String OFFSETS_VECTOR_NAME = "offsets";
- public final static String DATA_VECTOR_NAME = "data";
-
- private final static MaterializedField offsetsField =
- MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4));
-
- protected final UInt4Vector offsets;
- protected ValueVector vector;
-
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
- this(field, allocator, DEFAULT_DATA_VECTOR);
- }
-
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
- super(field, allocator);
- this.offsets = new UInt4Vector(offsetsField, allocator);
- this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
- }
-
- @Override
- public boolean allocateNewSafe() {
- if (!offsets.allocateNewSafe()) {
- return false;
- }
- offsets.zeroVector();
- return vector.allocateNewSafe();
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return offsets;
- }
-
- @Override
- public ValueVector getDataVector() {
- return vector;
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- offsets.setInitialCapacity(numRecords + 1);
- vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
- }
-
- @Override
- public int getValueCapacity() {
- final int offsetValueCapacity = offsets.getValueCapacity() - 1;
- if (vector == DEFAULT_DATA_VECTOR) {
- return offsetValueCapacity;
- }
- return Math.min(vector.getValueCapacity(), offsetValueCapacity);
- }
-
- @Override
- protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
- return super.getMetadataBuilder()
- .setGroupCount(getAccessor().getValueCount())
- .setValueCount(getAccessor().getInnerValueCount())
- .addChild(vector.getMetadata());
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- return offsets.getBufferSize() + vector.getBufferSize();
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return Collections.singleton(getDataVector()).iterator();
- }
-
- @Override
- public void clear() {
- offsets.clear();
- vector.clear();
- super.clear();
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), DrillBuf.class);
- if (clear) {
- for (DrillBuf buffer:buffers) {
- buffer.retain();
- }
- clear();
- }
- return buffers;
- }
-
- /**
- * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
- *
- * @see {@link ContainerVectorLike#size}
- */
- @Override
- public int size() {
- return vector == DEFAULT_DATA_VECTOR ? 0:1;
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- boolean created = false;
- if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
- final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
- vector = TypeHelper.getNewVector(field, allocator);
- // returned vector must have the same field
- assert field.equals(vector.getField());
- getField().addChild(field);
- created = true;
- }
-
- final TypeProtos.MajorType actual = vector.getField().getType();
- if (!actual.equals(descriptor.getType())) {
- final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
- descriptor.getType(), actual);
- throw new SchemaChangeRuntimeException(msg);
- }
-
- return new AddOrGetResult<>((T)vector, created);
- }
-
- public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
-
- @Override
- public int getValueCount() {
- return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
- }
-
- @Override
- public int getInnerValueCount() {
- return vector.getAccessor().getValueCount();
- }
-
- @Override
- public int getInnerValueCountAt(int index) {
- return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
- }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
-
- @Override
- public boolean isEmpty(int index) {
- return false;
- }
- }
-
- public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
-
- @Override
- public void startNewValue(int index) {
- offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
- setValueCount(index+1);
- }
-
- @Override
- public void setValueCount(int valueCount) {
- // TODO: populate offset end points
- offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
- final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
- vector.getMutator().setValueCount(childValueCount);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java
deleted file mode 100644
index 95e3365..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ContainerVectorLike.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-/**
- * A mix-in used for introducing container vector-like behaviour.
- */
-public interface ContainerVectorLike {
-
- /**
- * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
- *
- * @param descriptor vector descriptor
- * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
- * @throws org.apache.drill.common.exceptions.DrillRuntimeException
- * if schema change is not permissible between the given and existing data vector types.
- */
- <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
-
- /**
- * Returns the number of child vectors in this container vector-like instance.
- */
- int size();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java
deleted file mode 100644
index 450c673..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVectorLike.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-
-/**
- * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with
- * {@link org.apache.drill.exec.vector.RepeatedValueVector} subtypes.
- */
-public interface RepeatedFixedWidthVectorLike {
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount Number of separate repeating groupings.
- * @param innerValueCount Number of supported values in the vector.
- */
- public void allocateNew(int valueCount, int innerValueCount);
-
- /**
- * Load the records in the provided buffer based on the given number of values.
- * @param valueCount Number of separate repeating groupings.
- * @param innerValueCount Number atomic values the buffer contains.
- * @param buf Incoming buffer.
- * @return The number of bytes of the buffer that were consumed.
- */
- public int load(int valueCount, int innerValueCount, DrillBuf buf);
-
-// public interface RepeatedAccessor extends Accessor {
-// public int getGroupCount();
-// public int getValueCount();
-// public int getGroupSizeAtIndex(int index);
-// public ValueVector getAllChildValues();
-// }
-//
-// public interface RepeatedMutator extends Mutator {
-// public void setValueCounts(int parentValueCount, int childValueCount);
-// public void setRepetitionAtIndexSafe(int index, int repetitionCount);
-// public BaseDataValueVector getDataVector();
-// }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java
deleted file mode 100644
index 8e097e4..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-public interface RepeatedMutator extends ValueVector.Mutator {
- public void startNewGroup(int index);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java
deleted file mode 100644
index 95a7252..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedValueVector.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-/**
- * An abstraction representing repeated value vectors.
- *
- * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values).
- * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
- * uses the offset vector to determine the sequence of cells pertaining to an individual value.
- *
- */
-public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
-
- final static int DEFAULT_REPEAT_PER_RECORD = 5;
-
- /**
- * Returns the underlying offset vector or null if none exists.
- *
- * TODO(DRILL-2995): eliminate exposing low-level interfaces.
- */
- UInt4Vector getOffsetVector();
-
- /**
- * Returns the underlying data vector or null if none exists.
- */
- ValueVector getDataVector();
-
- @Override
- RepeatedAccessor getAccessor();
-
- @Override
- RepeatedMutator getMutator();
-
- interface RepeatedAccessor extends ValueVector.Accessor {
- /**
- * Returns total number of cells that vector contains.
- *
- * The result includes empty, null valued cells.
- */
- int getInnerValueCount();
-
-
- /**
- * Returns number of cells that the value at the given index contains.
- */
- int getInnerValueCountAt(int index);
-
- /**
- * Returns true if the value at the given index is empty, false otherwise.
- *
- * @param index value index
- */
- boolean isEmpty(int index);
- }
-
- interface RepeatedMutator extends ValueVector.Mutator {
- /**
- * Starts a new value that is a container of cells.
- *
- * @param index index of new value to start
- */
- void startNewValue(int index);
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java
deleted file mode 100644
index ac8589e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVectorLike.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-
-public interface RepeatedVariableWidthVectorLike {
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param totalBytes Desired size of the underlying data buffer.
- * @param parentValueCount Number of separate repeating groupings.
- * @param childValueCount Number of supported values in the vector.
- */
- public void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
-
- /**
- * Provide the maximum amount of variable width bytes that can be stored int his vector.
- * @return
- */
- public int getByteCapacity();
-
- /**
- * Load the records in the provided buffer based on the given number of values.
- * @param dataBytes The number of bytes associated with the data array.
- * @param parentValueCount Number of separate repeating groupings.
- * @param childValueCount Number of supported values in the vector.
- * @param buf Incoming buffer.
- * @return The number of bytes of the buffer that were consumed.
- */
- public int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
new file mode 100644
index 0000000..88b44db
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.SchemaChangeRuntimeException;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.ZeroVector;
+
+public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
+
+ public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+ public final static String OFFSETS_VECTOR_NAME = "offsets";
+ public final static String DATA_VECTOR_NAME = "data";
+
+ private final static MaterializedField offsetsField =
+ MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4));
+
+ protected final UInt4Vector offsets;
+ protected ValueVector vector;
+
+ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
+ this(field, allocator, DEFAULT_DATA_VECTOR);
+ }
+
+ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
+ super(field, allocator);
+ this.offsets = new UInt4Vector(offsetsField, allocator);
+ this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ offsets.zeroVector();
+ return vector.allocateNewSafe();
+ }
+
+ @Override
+ public UInt4Vector getOffsetVector() {
+ return offsets;
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ offsets.setInitialCapacity(numRecords + 1);
+ vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ final int offsetValueCapacity = offsets.getValueCapacity() - 1;
+ if (vector == DEFAULT_DATA_VECTOR) {
+ return offsetValueCapacity;
+ }
+ return Math.min(vector.getValueCapacity(), offsetValueCapacity);
+ }
+
+ @Override
+ protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+ return super.getMetadataBuilder()
+ .setGroupCount(getAccessor().getValueCount())
+ .setValueCount(getAccessor().getInnerValueCount())
+ .addChild(vector.getMetadata());
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return offsets.getBufferSize() + vector.getBufferSize();
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return Collections.singleton(getDataVector()).iterator();
+ }
+
+ @Override
+ public void clear() {
+ offsets.clear();
+ vector.clear();
+ super.clear();
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), DrillBuf.class);
+ if (clear) {
+ for (DrillBuf buffer:buffers) {
+ buffer.retain();
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+ /**
+ * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+ *
+ * @see {@link ContainerVectorLike#size}
+ */
+ @Override
+ public int size() {
+ return vector == DEFAULT_DATA_VECTOR ? 0:1;
+ }
+
+ @Override
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ boolean created = false;
+ if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
+ final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
+ vector = TypeHelper.getNewVector(field, allocator);
+ // returned vector must have the same field
+ assert field.equals(vector.getField());
+ getField().addChild(field);
+ created = true;
+ }
+
+ final TypeProtos.MajorType actual = vector.getField().getType();
+ if (!actual.equals(descriptor.getType())) {
+ final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+ descriptor.getType(), actual);
+ throw new SchemaChangeRuntimeException(msg);
+ }
+
+ return new AddOrGetResult<>((T)vector, created);
+ }
+
+ public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
+
+ @Override
+ public int getValueCount() {
+ return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+ }
+
+ @Override
+ public int getInnerValueCount() {
+ return vector.getAccessor().getValueCount();
+ }
+
+ @Override
+ public int getInnerValueCountAt(int index) {
+ return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty(int index) {
+ return false;
+ }
+ }
+
+ public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+
+ @Override
+ public void startNewValue(int index) {
+ offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
+ setValueCount(index+1);
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ // TODO: populate offset end points
+ offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+ final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+ vector.getMutator().setValueCount(childValueCount);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
new file mode 100644
index 0000000..999b47d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+
+/**
+ * A mix-in used for introducing container vector-like behaviour.
+ */
+public interface ContainerVectorLike {
+
+ /**
+ * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
+ *
+ * @param descriptor vector descriptor
+ * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
+ * @throws org.apache.drill.common.exceptions.DrillRuntimeException
+ * if schema change is not permissible between the given and existing data vector types.
+ */
+ <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
+
+ /**
+ * Returns the number of child vectors in this container vector-like instance.
+ */
+ int size();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
new file mode 100644
index 0000000..fb7ed2a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import io.netty.buffer.DrillBuf;
+
+/**
+ * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with
+ * {@link RepeatedValueVector} subtypes.
+ */
+public interface RepeatedFixedWidthVectorLike {
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount Number of separate repeating groupings.
+ * @param innerValueCount Number of supported values in the vector.
+ */
+ void allocateNew(int valueCount, int innerValueCount);
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param valueCount Number of separate repeating groupings.
+ * @param innerValueCount Number atomic values the buffer contains.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+ int load(int valueCount, int innerValueCount, DrillBuf buf);
+
+// public interface RepeatedAccessor extends Accessor {
+// public int getGroupCount();
+// public int getValueCount();
+// public int getGroupSizeAtIndex(int index);
+// public ValueVector getAllChildValues();
+// }
+//
+// public interface RepeatedMutator extends Mutator {
+// public void setValueCounts(int parentValueCount, int childValueCount);
+// public void setRepetitionAtIndexSafe(int index, int repetitionCount);
+// public BaseDataValueVector getDataVector();
+// }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index a5553b2..f6d3d88 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -29,21 +29,16 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.ComplexHolder;
import org.apache.drill.exec.expr.holders.RepeatedListHolder;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.AddOrGetResult;
-import org.apache.drill.exec.vector.BaseRepeatedValueVector;
-import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike;
import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.vector.RepeatedValueVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VectorDescriptor;
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index a97847b..2e12e55 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -43,10 +43,6 @@ import org.apache.drill.exec.util.CallBack;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.AddOrGetResult;
import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.BaseRepeatedValueVector;
-import org.apache.drill.exec.vector.RepeatedFixedWidthVectorLike;
-import org.apache.drill.exec.vector.RepeatedValueVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VectorDescriptor;
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
new file mode 100644
index 0000000..0fba292
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.ContainerVectorLike;
+
+/**
+ * An abstraction representing repeated value vectors.
+ *
+ * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values).
+ * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
+ * uses the offset vector to determine the sequence of cells pertaining to an individual value.
+ *
+ */
+public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+
+ final static int DEFAULT_REPEAT_PER_RECORD = 5;
+
+ /**
+ * Returns the underlying offset vector or null if none exists.
+ *
+ * TODO(DRILL-2995): eliminate exposing low-level interfaces.
+ */
+ UInt4Vector getOffsetVector();
+
+ /**
+ * Returns the underlying data vector or null if none exists.
+ */
+ ValueVector getDataVector();
+
+ @Override
+ RepeatedAccessor getAccessor();
+
+ @Override
+ RepeatedMutator getMutator();
+
+ interface RepeatedAccessor extends ValueVector.Accessor {
+ /**
+ * Returns total number of cells that vector contains.
+ *
+ * The result includes empty, null valued cells.
+ */
+ int getInnerValueCount();
+
+
+ /**
+ * Returns number of cells that the value at the given index contains.
+ */
+ int getInnerValueCountAt(int index);
+
+ /**
+ * Returns true if the value at the given index is empty, false otherwise.
+ *
+ * @param index value index
+ */
+ boolean isEmpty(int index);
+ }
+
+ interface RepeatedMutator extends ValueVector.Mutator {
+ /**
+ * Starts a new value that is a container of cells.
+ *
+ * @param index index of new value to start
+ */
+ void startNewValue(int index);
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/21de1381/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
new file mode 100644
index 0000000..c57143e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import io.netty.buffer.DrillBuf;
+
+public interface RepeatedVariableWidthVectorLike {
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param childValueCount Number of supported values in the vector.
+ */
+ void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
+
+ /**
+ * Provide the maximum amount of variable width bytes that can be stored int his vector.
+ * @return
+ */
+ int getByteCapacity();
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param dataBytes The number of bytes associated with the data array.
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param childValueCount Number of supported values in the vector.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+ int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf);
+}
[2/3] drill git commit: DRILL-3155: Part 2: Clear allocated memory
for composite vectors if one of the allocations fails
Posted by me...@apache.org.
DRILL-3155: Part 2: Clear allocated memory for composite vectors if one of the allocations fails
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/09e46df0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/09e46df0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/09e46df0
Branch: refs/heads/master
Commit: 09e46df0dada514aeca16cf80a71fa7179f1d8ce
Parents: 21de138
Author: Mehant Baid <me...@gmail.com>
Authored: Sat May 30 00:31:29 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Wed Jun 3 15:08:08 2015 -0700
----------------------------------------------------------------------
.../codegen/templates/NullableValueVectors.java | 61 ++++++++++++++++----
.../codegen/templates/RepeatedValueVectors.java | 52 ++++++++++++++---
.../templates/VariableLengthVectors.java | 46 ++++++++++-----
.../exec/vector/complex/AbstractMapVector.java | 20 ++++++-
.../vector/complex/BaseRepeatedValueVector.java | 20 ++++++-
.../exec/vector/complex/RepeatedMapVector.java | 33 ++++++++---
6 files changed, 185 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/09e46df0/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index 90ec6be..7f83542 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -124,8 +124,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public boolean allocateNewSafe() {
- if(!values.allocateNewSafe()) return false;
- if(!bits.allocateNewSafe()) return false;
+ /* Boolean to keep track if all the memory allocations were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if(!values.allocateNewSafe()) return false;
+ if(!bits.allocateNewSafe()) return false;
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
bits.zeroVector();
mutator.reset();
accessor.reset();
@@ -134,8 +147,13 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public void allocateNew(int totalBytes, int valueCount) {
- values.allocateNew(totalBytes, valueCount);
- bits.allocateNew(valueCount);
+ try {
+ values.allocateNew(totalBytes, valueCount);
+ bits.allocateNew(valueCount);
+ } catch(OutOfMemoryRuntimeException e){
+ clear();
+ throw e;
+ }
bits.zeroVector();
mutator.reset();
accessor.reset();
@@ -175,8 +193,13 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public void allocateNew() {
- values.allocateNew();
- bits.allocateNew();
+ try {
+ values.allocateNew();
+ bits.allocateNew();
+ } catch(OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
bits.zeroVector();
mutator.reset();
accessor.reset();
@@ -185,8 +208,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public boolean allocateNewSafe() {
- if(!values.allocateNewSafe()) return false;
- if(!bits.allocateNewSafe()) return false;
+ /* Boolean to keep track if all the memory allocations were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if(!values.allocateNewSafe()) return false;
+ if(!bits.allocateNewSafe()) return false;
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
bits.zeroVector();
mutator.reset();
accessor.reset();
@@ -195,8 +231,13 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public void allocateNew(int valueCount) {
- values.allocateNew(valueCount);
- bits.allocateNew(valueCount);
+ try {
+ values.allocateNew(valueCount);
+ bits.allocateNew(valueCount);
+ } catch(OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
bits.zeroVector();
mutator.reset();
accessor.reset();
http://git-wip-us.apache.org/repos/asf/drill/blob/09e46df0/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 813c3f8..12dce25 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -177,18 +177,36 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
}
- public boolean allocateNewSafe(){
- if(!offsets.allocateNewSafe()) return false;
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if(!offsets.allocateNewSafe()) return false;
+ if(!values.allocateNewSafe()) return false;
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
offsets.zeroVector();
- if(!values.allocateNewSafe()) return false;
mutator.reset();
return true;
}
public void allocateNew() {
- offsets.allocateNew();
+ try {
+ offsets.allocateNew();
+ values.allocateNew();
+ } catch (OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
offsets.zeroVector();
- values.allocateNew();
mutator.reset();
}
@@ -200,9 +218,14 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
}
public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
- offsets.allocateNew(valueCount+1);
+ try {
+ offsets.allocateNew(valueCount+1);
+ values.allocateNew(totalBytes, innerValueCount);
+ } catch (OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
offsets.zeroVector();
- values.allocateNew(totalBytes, innerValueCount);
mutator.reset();
}
@@ -230,9 +253,20 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
public void allocateNew(int valueCount, int innerValueCount) {
clear();
- offsets.allocateNew(valueCount+1);
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ offsets.allocateNew(valueCount+1);
+ values.allocateNew(innerValueCount);
+ } catch(OutOfMemoryRuntimeException e){
+ clear();
+ throw e;
+ }
offsets.zeroVector();
- values.allocateNew(innerValueCount);
mutator.reset();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/09e46df0/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index b3389e2..bd41e10 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -18,6 +18,7 @@
import java.lang.Override;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.vector.BaseDataValueVector;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
@@ -282,17 +283,28 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
allocationMonitor = 0;
}
- DrillBuf newBuf = allocator.buffer(allocationTotalByteCount);
- if(newBuf == null){
- return false;
+ /* Boolean to keep track if all the memory allocations were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ DrillBuf newBuf = allocator.buffer(allocationTotalByteCount);
+ if (newBuf == null) {
+ return false;
+ }
+ this.data = newBuf;
+ if (!offsetVector.allocateNewSafe()) {
+ return false;
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
}
-
- this.data = newBuf;
data.readerIndex(0);
-
- if(!offsetVector.allocateNewSafe()){
- return false;
- }
offsetVector.zeroVector();
return true;
}
@@ -300,15 +312,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public void allocateNew(int totalBytes, int valueCount) {
clear();
assert totalBytes >= 0;
- DrillBuf newBuf = allocator.buffer(totalBytes);
- if(newBuf == null){
- throw new OutOfMemoryRuntimeException(String.format("Failure while allocating buffer of %d bytes", totalBytes));
+ try {
+ DrillBuf newBuf = allocator.buffer(totalBytes);
+ if (newBuf == null) {
+ throw new OutOfMemoryRuntimeException(String.format("Failure while allocating buffer of %d bytes", totalBytes));
+ }
+ this.data = newBuf;
+ offsetVector.allocateNew(valueCount + 1);
+ } catch (OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
}
-
- this.data = newBuf;
data.readerIndex(0);
allocationTotalByteCount = totalBytes;
- offsetVector.allocateNew(valueCount+1);
offsetVector.zeroVector();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/09e46df0/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
index 3c01939..af364bd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -52,9 +52,23 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
@Override
public boolean allocateNewSafe() {
- for (ValueVector v : vectors.values()) {
- if (!v.allocateNewSafe()) {
- return false;
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+
+ for (ValueVector v : vectors.values()) {
+ if (!v.allocateNewSafe()) {
+ return false;
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
}
}
return true;
http://git-wip-us.apache.org/repos/asf/drill/blob/09e46df0/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
index 88b44db..f292e4c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -61,13 +61,27 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
@Override
public boolean allocateNewSafe() {
- if (!offsets.allocateNewSafe()) {
- return false;
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = vector.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
}
offsets.zeroVector();
- return vector.allocateNewSafe();
+ return success;
}
+
@Override
public UInt4Vector getOffsetVector() {
return offsets;
http://git-wip-us.apache.org/repos/asf/drill/blob/09e46df0/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 2e12e55..4617ede 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -35,6 +35,7 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.ComplexHolder;
import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -103,11 +104,16 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu
@Override
public void allocateNew(int groupCount, int innerValueCount) {
clear();
- offsets.allocateNew(groupCount+1);
- offsets.zeroVector();
- for (ValueVector v : getChildren()) {
- AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
+ try {
+ offsets.allocateNew(groupCount + 1);
+ for (ValueVector v : getChildren()) {
+ AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
+ }
+ } catch (OutOfMemoryRuntimeException e){
+ clear();
+ throw e;
}
+ offsets.zeroVector();
mutator.reset();
}
@@ -216,11 +222,24 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu
@Override
public boolean allocateNewSafe() {
- if (!offsets.allocateNewSafe()) {
- return false;
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = super.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
}
offsets.zeroVector();
- return super.allocateNewSafe();
+ return success;
}
protected static class SingleMapTransferPair implements TransferPair {
[3/3] drill git commit: DRILL-3245: Fix error message for unsupported
aggregate functions on varchar data type
Posted by me...@apache.org.
DRILL-3245: Fix error message for unsupported aggregate functions on varchar data type
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/287f52db
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/287f52db
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/287f52db
Branch: refs/heads/master
Commit: 287f52db0bd0125e0ad1b3408f22775224ee9494
Parents: 09e46df
Author: Mehant Baid <me...@gmail.com>
Authored: Wed Jun 3 15:12:30 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Wed Jun 3 15:12:30 2015 -0700
----------------------------------------------------------------------
.../expr/fn/impl/AggregateErrorFunctions.java | 87 ++++++++++++++++++--
1 file changed, 82 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/287f52db/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AggregateErrorFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AggregateErrorFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AggregateErrorFunctions.java
index a95a1c3..8161a43 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AggregateErrorFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AggregateErrorFunctions.java
@@ -25,10 +25,18 @@ import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+/*
+ * TODO: For a handful of functions this approach of using function binding to detect that it is an invalid function is okay.
+ * However moving forward we should introduce a validation phase after we learn the data types and before we try
+ * to perform function resolution. Otherwise with implicit cast we will try to bind to an existing function.
+ */
public class AggregateErrorFunctions {
- @FunctionTemplate(names = {"sum", "max", "avg", "stddev_pop", "stddev_samp", "stddev", "var_pop", "var_samp", "variance"}, scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+ @FunctionTemplate(names = {"sum", "max", "avg", "stddev_pop", "stddev_samp", "stddev", "var_pop",
+ "var_samp", "variance"}, scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
public static class BitAggregateErrorFunctions implements DrillAggFunc {
@Param BitHolder in;
@@ -37,7 +45,10 @@ public class AggregateErrorFunctions {
public void setup() {
if (true) {
- throw new RuntimeException("Only COUNT aggregate function supported for Boolean type");
+ throw org.apache.drill.common.exceptions.UserException
+ .unsupportedError()
+ .message("Only COUNT aggregate function supported for Boolean type")
+ .build();
}
}
@@ -55,8 +66,9 @@ public class AggregateErrorFunctions {
}
- @FunctionTemplate(names = {"sum", "max", "avg", "stddev_pop", "stddev_samp", "stddev", "var_pop", "var_samp", "variance"}, scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
- public static class NullableBitAggregateErrorFunctions implements DrillAggFunc{
+ @FunctionTemplate(names = {"sum", "max", "avg", "stddev_pop", "stddev_samp", "stddev", "var_pop",
+ "var_samp", "variance"}, scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+ public static class NullableBitAggregateErrorFunctions implements DrillAggFunc {
@Param NullableBitHolder in;
@Workspace BigIntHolder value;
@@ -64,7 +76,72 @@ public class AggregateErrorFunctions {
public void setup() {
if (true) {
- throw new RuntimeException("Only COUNT aggregate function supported for Boolean type");
+ throw org.apache.drill.common.exceptions.UserException
+ .unsupportedError()
+ .message("Only COUNT aggregate function supported for Boolean type")
+ .build();
+ }
+ }
+
+ @Override
+ public void add() {
+ }
+
+ @Override
+ public void output() {
+ }
+
+ @Override
+ public void reset() {
+ }
+ }
+
+
+ @FunctionTemplate(names = {"sum", "avg", "stddev_pop", "stddev_samp", "stddev", "var_pop", "var_samp", "variance"},
+ scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+ public static class VarCharAggregateErrorFunctions implements DrillAggFunc {
+
+ @Param VarCharHolder in;
+ @Workspace BigIntHolder value;
+ @Output BigIntHolder out;
+
+ public void setup() {
+ if (true) {
+ throw org.apache.drill.common.exceptions.UserException
+ .unsupportedError()
+ .message("Only COUNT, MIN and MAX aggregate functions supported for VarChar type")
+ .build();
+ }
+ }
+
+ @Override
+ public void add() {
+ }
+
+ @Override
+ public void output() {
+ }
+
+ @Override
+ public void reset() {
+ }
+
+ }
+
+ @FunctionTemplate(names = {"sum", "avg", "stddev_pop", "stddev_samp", "stddev", "var_pop", "var_samp", "variance"},
+ scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+ public static class NullableVarCharAggregateErrorFunctions implements DrillAggFunc {
+
+ @Param NullableVarCharHolder in;
+ @Workspace BigIntHolder value;
+ @Output BigIntHolder out;
+
+ public void setup() {
+ if (true) {
+ throw org.apache.drill.common.exceptions.UserException
+ .unsupportedError()
+ .message("Only COUNT, MIN and MAX aggregate functions supported for VarChar type")
+ .build();
}
}