You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2018/04/07 12:35:53 UTC
[07/11] drill git commit: DRILL-6230: Extend row set readers to
handle hyper vectors
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
deleted file mode 100644
index 2056220..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.test.rowSet.test;
-
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VarCharVector;
-
-import com.google.common.base.Charsets;
-
-/**
- * Handy tool to visualize string and offset vectors for
- * debugging.
- */
-
-public class VectorPrinter {
-
- public static void printOffsets(UInt4Vector vector, int start, int length) {
- header(vector, start, length);
- for (int i = start, j = 0; j < length; i++, j++) {
- if (j > 0) {
- System.out.print(" ");
- }
- System.out.print(vector.getAccessor().get(i));
- }
- System.out.print("], addr = ");
- System.out.println(vector.getBuffer().addr());
- }
-
- public static void printStrings(VarCharVector vector, int start, int length) {
- printOffsets(vector.getOffsetVector(), start, length + 1);
- header(vector, start, length);
- System.out.println();
- for (int i = start, j = 0; j < length; i++, j++) {
- System.out.print(" ");
- System.out.print(i);
- System.out.print(": \"");
- System.out.print(stringAt(vector, i));
- System.out.println("\"");
- }
- System.out.println("]");
- }
-
- public static void header(ValueVector vector, int start, int length) {
- System.out.print(vector.getClass());
- System.out.print(": (");
- System.out.print(start);
- System.out.print(" - ");
- System.out.print(start + length - 1);
- System.out.print("): [");
- }
-
- public static String stringAt(VarCharVector vector, int i) {
- return new String(vector.getAccessor().get(i), Charsets.UTF_8);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
index 109500a..513f50b 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
@@ -823,4 +823,19 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
historicalLog.buildHistory(sb, indent + 1, verbosity.includeStackTraces);
}
}
+
+ /**
+ * Convenience method to read buffer bytes into a newly allocated byte
+ * array.
+ *
+ * @param srcOffset the offset into this buffer of the data to read
+ * @param length number of bytes to read
+ * @return byte array with the requested bytes
+ */
+
+ public byte[] unsafeGetMemory(int srcOffset, int length) {
+ byte buf[] = new byte[length];
+ PlatformDependent.copyMemory(addr + srcOffset, buf, 0, length);
+ return buf;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/codegen/templates/ColumnAccessors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
index 14ec1e8..4836099 100644
--- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java
+++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
@@ -1,3 +1,4 @@
+<#macro copyright>
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -16,9 +17,10 @@
* limitations under the License.
*/
+// This class is generated using Freemarker and the ${.template_name} template.
+</#macro>
<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/drill/exec/vector/accessor/ColumnAccessors.java" />
-<#include "/@includes/license.ftl" />
<#macro getType drillType label>
@Override
public ValueType valueType() {
@@ -31,74 +33,6 @@
</#if>
}
</#macro>
-<#macro bindReader vectorPrefix drillType isArray >
- <#if drillType = "Decimal9" || drillType == "Decimal18">
- private MajorType type;
- </#if>
- private ${vectorPrefix}${drillType}Vector.Accessor accessor;
-
- @Override
- public void bindVector(ValueVector vector) {
- <#if drillType = "Decimal9" || drillType == "Decimal18">
- type = vector.getField().getType();
- </#if>
- accessor = ((${vectorPrefix}${drillType}Vector) vector).getAccessor();
- }
-
- <#if drillType = "Decimal9" || drillType == "Decimal18">
- @Override
- public void bindVector(MajorType type, VectorAccessor va) {
- super.bindVector(type, va);
- this.type = type;
- }
-
- </#if>
- private ${vectorPrefix}${drillType}Vector.Accessor accessor() {
- if (vectorAccessor == null) {
- return accessor;
- } else {
- return ((${vectorPrefix}${drillType}Vector) vectorAccessor.vector()).getAccessor();
- }
- }
-</#macro>
-<#macro get drillType accessorType label isArray>
- @Override
- public ${accessorType} get${label}(<#if isArray>int index</#if>) {
- <#assign getObject ="getObject"/>
- <#if isArray>
- <#assign indexVar = "index"/>
- <#else>
- <#assign indexVar = ""/>
- </#if>
- <#if drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary">
- return accessor().get(vectorIndex.vectorIndex(${indexVar}));
- <#elseif drillType == "Decimal9" || drillType == "Decimal18">
- return DecimalUtility.getBigDecimalFromPrimitiveTypes(
- accessor().get(vectorIndex.vectorIndex(${indexVar})),
- type.getScale(),
- type.getPrecision());
- <#elseif accessorType == "BigDecimal" || accessorType == "Period">
- return accessor().${getObject}(vectorIndex.vectorIndex(${indexVar}));
- <#elseif drillType == "UInt1">
- return ((int) accessor().get(vectorIndex.vectorIndex(${indexVar}))) & 0xFF;
- <#else>
- return accessor().get(vectorIndex.vectorIndex(${indexVar}));
- </#if>
- }
- <#if drillType == "VarChar">
-
- @Override
- public String getString(<#if isArray>int index</#if>) {
- return new String(getBytes(${indexVar}), Charsets.UTF_8);
- }
- <#elseif drillType == "Var16Char">
-
- @Override
- public String getString(<#if isArray>int index</#if>) {
- return new String(getBytes(${indexVar}), Charsets.UTF_16);
- }
- </#if>
-</#macro>
<#macro build types vectorType accessorType>
<#if vectorType == "Repeated">
<#assign fnPrefix = "Array" />
@@ -126,24 +60,27 @@
</#list>
}
</#macro>
+<@copyright />
package org.apache.drill.exec.vector.accessor;
import java.math.BigDecimal;
import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.vector.DateUtilities;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.*;
import org.apache.drill.exec.util.DecimalUtility;
-import org.apache.drill.exec.vector.accessor.reader.BaseScalarReader;
-import org.apache.drill.exec.vector.accessor.reader.BaseElementReader;
+import org.apache.drill.exec.vector.accessor.reader.BaseScalarReader.BaseVarWidthReader;
+import org.apache.drill.exec.vector.accessor.reader.BaseScalarReader.BaseFixedWidthReader;
import org.apache.drill.exec.vector.accessor.reader.VectorAccessor;
-import org.apache.drill.exec.vector.accessor.writer.BaseScalarWriter;
import org.apache.drill.exec.vector.accessor.writer.AbstractFixedWidthWriter.BaseFixedWidthWriter;
import org.apache.drill.exec.vector.accessor.writer.BaseVarWidthWriter;
import com.google.common.base.Charsets;
+import io.netty.buffer.DrillBuf;
+
import org.joda.time.Period;
/**
@@ -161,8 +98,6 @@ import org.joda.time.Period;
* row.)
*/
-// This class is generated using freemarker and the ${.template_name} template.
-
public class ColumnAccessors {
<#list vv.types as type>
@@ -177,56 +112,141 @@ public class ColumnAccessors {
<#if accessorType=="BigDecimal">
<#assign label="Decimal">
</#if>
- <#if drillType == "VarChar" || drillType == "Var16Char">
+ <#assign varWidth = drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary" />
+ <#assign decimal = drillType == "Decimal9" || drillType == "Decimal18" ||
+ drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse" />
+ <#if varWidth>
<#assign accessorType = "byte[]">
<#assign label = "Bytes">
+ <#assign putArgs = ", int len">
+ <#else>
+ <#assign putArgs = "">
+ </#if>
+ <#if javaType == "char">
+ <#assign putType = "short" />
+ <#assign doCast = true />
+ <#else>
+ <#assign putType = javaType />
+ <#assign doCast = (cast == "set") />
</#if>
<#if ! notyet>
//------------------------------------------------------------------------
// ${drillType} readers and writers
- public static class ${drillType}ColumnReader extends BaseScalarReader {
+ <#if varWidth>
+ public static class ${drillType}ColumnReader extends BaseVarWidthReader {
+
+ <#else>
+ public static class ${drillType}ColumnReader extends BaseFixedWidthReader {
+
+ private static final int VALUE_WIDTH = ${drillType}Vector.VALUE_WIDTH;
- <@bindReader "" drillType false />
+ <#if decimal>
+ private MajorType type;
+
+ </#if>
+ </#if>
+ <#if decimal>
+ @Override
+ public void bindVector(ColumnMetadata schema, VectorAccessor va) {
+ super.bindVector(schema, va);
+ <#if decimal>
+ type = va.type();
+ </#if>
+ }
+ </#if>
<@getType drillType label />
- <@get drillType accessorType label false/>
- }
-
- public static class Nullable${drillType}ColumnReader extends BaseScalarReader {
-
- <@bindReader "Nullable" drillType false />
-
- <@getType drillType label />
+ <#if ! varWidth>
+ @Override public int width() { return VALUE_WIDTH; }
+ </#if>
@Override
- public boolean isNull() {
- return accessor().isNull(vectorIndex.vectorIndex());
+ public ${accessorType} get${label}() {
+ <#assign getObject ="getObject"/>
+ <#assign indexVar = ""/>
+ final DrillBuf buf = bufferAccessor.buffer();
+ <#if ! varWidth>
+ final int readOffset = vectorIndex.offset();
+ <#assign getOffset = "readOffset * VALUE_WIDTH">
+ </#if>
+ <#if varWidth>
+ final long entry = offsetsReader.getEntry();
+ return buf.unsafeGetMemory((int) (entry >> 32), (int) (entry & 0xFFFF_FFFF));
+ <#elseif drillType == "Decimal9">
+ return DecimalUtility.getBigDecimalFromPrimitiveTypes(
+ buf.getInt(${getOffset}),
+ type.getScale(),
+ type.getPrecision());
+ <#elseif drillType == "Decimal18">
+ return DecimalUtility.getBigDecimalFromPrimitiveTypes(
+ buf.getLong(${getOffset}),
+ type.getScale(),
+ type.getPrecision());
+ <#elseif drillType == "IntervalYear">
+ return DateUtilities.fromIntervalYear(
+ buf.getInt(${getOffset}));
+ <#elseif drillType == "IntervalDay">
+ final int offset = ${getOffset};
+ return DateUtilities.fromIntervalDay(
+ buf.getInt(offset),
+ buf.getInt(offset + ${minor.millisecondsOffset}));
+ <#elseif drillType == "Interval">
+ final int offset = ${getOffset};
+ return DateUtilities.fromInterval(
+ buf.getInt(offset),
+ buf.getInt(offset + ${minor.daysOffset}),
+ buf.getInt(offset + ${minor.millisecondsOffset}));
+ <#elseif drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse">
+ return DecimalUtility.getBigDecimalFromSparse(buf, ${getOffset},
+ ${minor.nDecimalDigits}, type.getScale());
+ <#elseif drillType == "Decimal28Dense" || drillType == "Decimal38Dense">
+ return DecimalUtility.getBigDecimalFromDense(buf, ${getOffset},
+ ${minor.nDecimalDigits}, type.getScale(),
+ ${minor.maxPrecisionDigits}, VALUE_WIDTH);
+ <#elseif drillType == "UInt1">
+ return buf.getByte(${getOffset}) & 0xFF;
+ <#elseif drillType == "UInt2">
+ return buf.getShort(${getOffset}) & 0xFFFF;
+ <#elseif drillType == "UInt4">
+ // Should be the following:
+ // return ((long) buf.unsafeGetInt(${getOffset})) & 0xFFFF_FFFF;
+ // else, the unsigned values of 32 bits are mapped to negative.
+ return buf.getInt(${getOffset});
+ <#elseif drillType == "Float4">
+ return Float.intBitsToFloat(buf.getInt(${getOffset}));
+ <#elseif drillType == "Float8">
+ return Double.longBitsToDouble(buf.getLong(${getOffset}));
+ <#else>
+ return buf.get${putType?cap_first}(${getOffset});
+ </#if>
}
+ <#if drillType == "VarChar">
- <@get drillType accessorType label false />
- }
-
- public static class Repeated${drillType}ColumnReader extends BaseElementReader {
-
- <@bindReader "" drillType true />
-
- <@getType drillType label />
+ @Override
+ public String getString() {
+ return new String(getBytes(${indexVar}), Charsets.UTF_8);
+ }
+ <#elseif drillType == "Var16Char">
- <@get drillType accessorType label true />
+ @Override
+ public String getString() {
+ return new String(getBytes(${indexVar}), Charsets.UTF_16);
+ }
+ </#if>
}
- <#assign varWidth = drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary" />
<#if varWidth>
public static class ${drillType}ColumnWriter extends BaseVarWidthWriter {
<#else>
public static class ${drillType}ColumnWriter extends BaseFixedWidthWriter {
- <#if drillType = "Decimal9" || drillType == "Decimal18" ||
- drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse">
+
+ private static final int VALUE_WIDTH = ${drillType}Vector.VALUE_WIDTH;
+
+ <#if decimal>
private MajorType type;
</#if>
- private static final int VALUE_WIDTH = ${drillType}Vector.VALUE_WIDTH;
</#if>
private final ${drillType}Vector vector;
@@ -234,8 +254,7 @@ public class ColumnAccessors {
<#if varWidth>
super(((${drillType}Vector) vector).getOffsetVector());
<#else>
- <#if drillType = "Decimal9" || drillType == "Decimal18" ||
- drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse">
+ <#if decimal>
type = vector.getField().getType();
</#if>
</#if>
@@ -300,12 +319,12 @@ public class ColumnAccessors {
<#elseif drillType == "IntervalDay">
final int offset = ${putAddr};
drillBuf.setInt(offset, value.getDays());
- drillBuf.setInt(offset + 4, periodToMillis(value));
+ drillBuf.setInt(offset + 4, DateUtilities.periodToMillis(value));
<#elseif drillType == "Interval">
final int offset = ${putAddr};
drillBuf.setInt(offset, value.getYears() * 12 + value.getMonths());
drillBuf.setInt(offset + 4, value.getDays());
- drillBuf.setInt(offset + 8, periodToMillis(value));
+ drillBuf.setInt(offset + 8, DateUtilities.periodToMillis(value));
<#elseif drillType == "Float4">
drillBuf.setInt(${putAddr}, Float.floatToRawIntBits((float) value));
<#elseif drillType == "Float8">
@@ -335,18 +354,22 @@ public class ColumnAccessors {
</#if>
</#list>
</#list>
- public static int periodToMillis(Period value) {
- return ((value.getHours() * 60 +
- value.getMinutes()) * 60 +
- value.getSeconds()) * 1000 +
- value.getMillis();
- }
+}
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/accessor/ColumnAccessorUtils.java" />
+<@copyright />
-<@build vv.types "Required" "Reader" />
+package org.apache.drill.exec.vector.accessor;
-<@build vv.types "Nullable" "Reader" />
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.vector.accessor.ColumnAccessors.*;
+import org.apache.drill.exec.vector.accessor.reader.BaseScalarReader;
+import org.apache.drill.exec.vector.accessor.writer.BaseScalarWriter;
-<@build vv.types "Repeated" "Reader" />
+public class ColumnAccessorUtils {
+
+ private ColumnAccessorUtils() { }
+
+<@build vv.types "Required" "Reader" />
<@build vv.types "Required" "Writer" />
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayReader.java
index 8f33f0e..0679c3b 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ArrayReader.java
@@ -32,7 +32,7 @@ package org.apache.drill.exec.vector.accessor;
* {@see ArrayWriter}
*/
-public interface ArrayReader {
+public interface ArrayReader extends ColumnReader {
/**
* Number of elements in the array.
@@ -50,26 +50,6 @@ public interface ArrayReader {
ObjectType entryType();
/**
- * Return a reader for the elements of a scalar array.
- * @return reader for scalar elements
- */
-
- ScalarElementReader elements();
-
- /**
- * Return a generic object reader for the array entry. Not available
- * for scalar elements. Positions the reader to read the selected
- * element.
- *
- * @param index array index
- * @return generic object reader
- */
-
- ObjectReader entry(int index);
- TupleReader tuple(int index);
- ArrayReader array(int index);
-
- /**
* Return the generic object reader for the array element. This
* version <i>does not</i> position the reader, the client must
* call {@link setPosn()} to set the position. This form allows
@@ -77,6 +57,7 @@ public interface ArrayReader {
*/
ObjectReader entry();
+ ScalarReader scalar();
TupleReader tuple();
ArrayReader array();
@@ -88,19 +69,14 @@ public interface ArrayReader {
void setPosn(int index);
- /**
- * Return the entire array as an <tt>List</tt> of objects.
- * Note, even if the array is scalar, the elements are still returned
- * as a list. This method is primarily for testing.
- * @return array as a <tt>List</tt> of objects
- */
-
- Object getObject();
+ void rewind();
/**
- * Return the entire array as a string. Primarily for debugging.
- * @return string representation of the array
+ * Move forward one position.
+ *
+ * @return true if another position is available, false if
+ * the end of the array is reached
*/
- String getAsString();
+ boolean next();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReader.java
new file mode 100644
index 0000000..15e5c74
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+
+/**
+ * Base interface for all column readers, defining a generic set of methods
+ * that all readers provide. In particular, given the metadata and the object
+ * type, one can determine what to do with each reader when working with readers
+ * generically. The <tt>getObject()</tt> and <tt>getAsString()</tt> methods provide
+ * generic data access for tests and debugging.
+ */
+
+public interface ColumnReader {
+
+ ColumnMetadata schema();
+
+ /**
+ * The type of this reader.
+ *
+ * @return type of reader
+ */
+
+ ObjectType type();
+
+ /**
+ * Determine if this value is null.
+ * <ul>
+ * <li>Nullable scalar: determine if the value is null.</li>
+ * <li>Non-nullable scalar: always returns <tt>false</tt>.</li>
+ * <li>Arrays: always returns </tt>false</tt.></li>
+ * <li>Lists: determine if the list for the current row is null.
+ * In a list, an array entry can be null, empty, or can contain
+ * items. In repeated types, the array itself is never null.
+ * If the array is null, then it implicitly has no entries.</li>
+ * <li>Map or Repeated Map: Always returns <tt>false</tt>.</li>
+ * <li>Map inside a union, or in a list that contains a union,
+ * the tuple itself can be null.</li>
+ * <li>Union: Determine if the current value is null. Null values have no type
+ * and no associated reader.</li>
+ * </ul>
+ *
+ * @return <tt>true</tt> if this value is null; <tt>false</tt> otherwise
+ */
+
+ boolean isNull();
+
+ /**
+ * Return the value of the underlying data as a Java object.
+ * Primarily for testing
+ * <ul>
+ * <li>Array: Return the entire array as an <tt>List</tt> of objects.
+ * Note, even if the array is scalar, the elements are still returned
+ * as a list.</li>
+ * </ul>
+ * @return the value as a Java object
+ */
+
+ Object getObject();
+
+ /**
+ * Return the entire object as a string. Primarily for debugging.
+ * @return string representation of the object
+ */
+
+ String getAsString();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReaderIndex.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReaderIndex.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReaderIndex.java
index b40b705..edc3623 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReaderIndex.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnReaderIndex.java
@@ -18,11 +18,105 @@
package org.apache.drill.exec.vector.accessor;
/**
- * Index into a vector batch, or an array, at read time.
- * Supports direct, indirect and hyper-batches.
+ * The reader structure is heavily recursive. The top-level reader
+ * iterates over batches, possibly through an indirection vector
+ * (SV2 or SV4.) The row is tuple of top-level vectors. Each top-level
+ * vector may be an array. Iteration through the array works identically
+ * to iteration over the batch as a whole. (In fact, the scalar readers
+ * don't know if they are top-level or array readers.) Array nesting
+ * can continue to any level of depth.
+ * <p>
+ * Further, when used with a logical join, the top-level iteration
+ * may be over an array, with an implicit join out to enclosing nesting
+ * levels.
+ * <p>
+ * Because of this, the same index interface must work at all nesting
+ * levels: at the top, and within arrays. This interface
+ * supports a usage model as follows:<pre><code>
+ * ColumnReaderIndex index = ...
+ * while (index.hasNext()) {
+ * index.next();
+ * int hyperIndex = index.hyperVectorIndex();
+ * int vectorOffset = index.offset();
+ * }</code></pre>
+ * <p>
+ * When convenient, the following abbreviated form is also
+ * supported:<pre><code>
+ * ColumnReaderIndex index = ...
+ * while (index.next()) {
+ * int hyperIndex = index.hyperVectorIndex();
+ * int vectorOffset = index.offset();
+ * }</code></pre>
+ * <p>
+ * For a top-level index, the check of <tt>hasNext()</tt> and
+ * call to <tt>next()</tt> is done by the row set reader. For
+ * arrays, the call to <tt>hasNext()</tt> is done by the array
+ * reader. The call to <tt>next()</tt> is done by the scalar
+ * reader (for scalar arrays) or the array reader (for other
+ * arrays.)
+ * <p>
+ * The hyper-vector index has meaning only for top-level vectors,
+ * and is ignored by nested vectors. (Nested vectors work by navigating
+ * down from a top-level vector.) But, as noted above, any given
+ * reader does not know if it is at the top or nested level, instead
+ * it is the {@link VectorAccessor} abstraction that works out the
+ * nesting levels.
*/
public interface ColumnReaderIndex {
- int batchIndex();
- int vectorIndex();
+
+ /**
+ * Ordinal index within the batch or array. Increments from -1.
+ * (The position before the first item.)
+ * Identifies the logical row number of top-level records,
+ * or the array element for arrays. Actual physical
+ * index may be different if an indirection layer is in use.
+ *
+ * @return logical read index
+ */
+
+ int logicalIndex();
+
+ /**
+ * When used with a hyper-vector (SV4) based batch, returns the
+ * index of the current batch within the hyper-batch. If this is
+ * a single batch, or a nested index, then always returns 0.
+ *
+ * @return batch index of the current row within the
+ * hyper-batch
+ */
+
+ int hyperVectorIndex();
+
+ /**
+ * Vector offset to read. For top-level vectors, the offset may be
+ * through an indirection (SV2 or SV4). For arrays, the offset is the
+ * absolute position, with the vector of the current array element.
+ *
+ * @return vector read index
+ */
+
+ int offset();
+
+ /**
+ * Advances the index to the next position. Used:
+ * <ul>
+ * <li>At the top level for normal readers or</li>
+ * <liAt a nested level for implicit join readers, and</li>
+ * <li>An each array level to iterate over arrays.</li>
+ * </ul>
+ *
+ * @return true if another value is available, false if EOF
+ */
+
+ boolean next();
+
+ /**
+ * Return the number of items that this index indexes: top-level record
+ * count for the root index; total element count for nested arrays.
+ *
+ * @return element count at this index level
+ */
+
+ int size();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectReader.java
index 9c53e58..e3527c5 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectReader.java
@@ -29,32 +29,8 @@ package org.apache.drill.exec.vector.accessor;
* {@see ObjectWriter>
*/
-public interface ObjectReader {
-
- /**
- * The type of this reader.
- *
- * @return type of reader
- */
-
- ObjectType type();
+public interface ObjectReader extends ColumnReader {
ScalarReader scalar();
- ScalarElementReader elements();
TupleReader tuple();
ArrayReader array();
-
- /**
- * Return the value of the underlying data as a Java object.
- * Primarily for testing
- * @return Java object that represents the underlying value
- */
-
- Object getObject();
-
- /**
- * Return the entire object as a string. Primarily for debugging.
- * @return string representation of the object
- */
-
- String getAsString();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarElementReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarElementReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarElementReader.java
deleted file mode 100644
index d1f31a8..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarElementReader.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor;
-
-import java.math.BigDecimal;
-
-import org.joda.time.Period;
-
-/**
- * Interface to access the values of an array column. In general, each
- * vector implements just one of the get methods. Check the vector type
- * to know which method to use. Though, generally, when writing test
- * code, the type is known to the test writer.
- * <p>
- * Arrays allow random access to the values within the array. The index
- * passed to each method is the index into the array for the current
- * row and column. (This means that arrays are three dimensional:
- * the usual (row, column) dimensions plus an array index dimension:
- * (row, column, array index).
- * <p>
- * Note that the <tt>isNull()</tt> method is provided for completeness,
- * but no Drill array allows null values at present.
- * <p>
- * {@see ScalarWriter}
- */
-
-public interface ScalarElementReader {
- /**
- * Describe the type of the value. This is a compression of the
- * value vector type: it describes which method will return the
- * vector value.
- * @return the value type which indicates which get method
- * is valid for the column
- */
-
- ValueType valueType();
- int size();
-
- boolean isNull(int index);
- int getInt(int index);
- long getLong(int index);
- double getDouble(int index);
- String getString(int index);
- byte[] getBytes(int index);
- BigDecimal getDecimal(int index);
- Period getPeriod(int index);
-
- Object getObject(int index);
- String getAsString(int index);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
index e1c26bf..5b09039 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
@@ -44,7 +44,7 @@ import org.joda.time.Period;
* {@see ScalarWriter}
*/
-public interface ScalarReader {
+public interface ScalarReader extends ColumnReader {
/**
* Describe the type of the value. This is a compression of the
* value vector type: it describes which method will return the
@@ -54,14 +54,6 @@ public interface ScalarReader {
*/
ValueType valueType();
-
- /**
- * Report if the column is null. Non-nullable columns always
- * return <tt>false</tt>.
- * @return true if the column value is null, false if the
- * value is set
- */
- boolean isNull();
int getInt();
long getLong();
double getDouble();
@@ -69,7 +61,4 @@ public interface ScalarReader {
byte[] getBytes();
BigDecimal getDecimal();
Period getPeriod();
-
- Object getObject();
- String getAsString();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
index 8d691c3..c33f579 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
@@ -25,15 +25,33 @@ import org.apache.drill.exec.record.metadata.TupleMetadata;
* by name or column index (as defined in the tuple schema.)
* Also provides two generic methods to get the value as a
* Java object or as a string.
- * <p>
- * {@see TupleWriter}
+ *
+ * @see {@link TupleWriter}
*/
-public interface TupleReader {
- TupleMetadata schema();
+public interface TupleReader extends ColumnReader {
+ TupleMetadata tupleSchema();
int columnCount();
+ /**
+ * Return a column reader by column index as reported by the
+ * associated metadata.
+ *
+ * @param colIndex column index
+ * @return reader for the column
+ * @throws IndexOutOfRangeException if the index is invalid
+ */
+
ObjectReader column(int colIndex);
+
+ /**
+ * Return a column reader by name.
+ *
+ * @param colIndex column name
+ * @return reader for the column, or <tt>null</tt> if no such
+ * column exists
+ */
+
ObjectReader column(String colName);
// Convenience methods
@@ -46,9 +64,4 @@ public interface TupleReader {
TupleReader tuple(String colName);
ArrayReader array(int colIndex);
ArrayReader array(String colName);
- ScalarElementReader elements(int colIndex);
- ScalarElementReader elements(String colName);
-
- Object getObject();
- String getAsString();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java
new file mode 100644
index 0000000..dee2612
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+
+/**
+ * Raised when a column accessor reads or writes the value using the wrong
+ * Java type (which may indicate an data inconsistency in the input data.)
+ */
+
+public class UnsupportedConversionError extends UnsupportedOperationException {
+
+ private static final long serialVersionUID = 1L;
+
+ private UnsupportedConversionError(String message) {
+ super(message);
+ }
+
+ public static UnsupportedConversionError readError(ColumnMetadata schema, String javaType) {
+ return new UnsupportedConversionError(
+ String.format("Column `%s`: Unsupported conversion from Drill type %s to Java type %s",
+ schema.name(), schema.type().name(), javaType));
+ }
+
+ public static UnsupportedConversionError writeError(ColumnMetadata schema, String javaType) {
+ return new UnsupportedConversionError(
+ String.format("Column `%s`: Unsupported conversion from Java type %s to Drill type %s",
+ schema.name(), schema.type().name(), javaType));
+ }
+
+ public static UnsupportedConversionError nullError(ColumnMetadata schema) {
+ return new UnsupportedConversionError(
+ String.format("Column `%s`: Type %s %s is not nullable",
+ schema.name(), schema.mode().name(), schema.type().name()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueType.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueType.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueType.java
index e6687dc..5059977 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueType.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueType.java
@@ -27,5 +27,60 @@ package org.apache.drill.exec.vector.accessor;
*/
public enum ValueType {
- INTEGER, LONG, DOUBLE, STRING, BYTES, DECIMAL, PERIOD
+
+ /**
+ * The value is set from an integer: TINYINT,
+ * SMALLINT, INT, UINT1, and UINT2.
+ */
+
+ INTEGER,
+
+ /**
+ * The value set from a long: BIGINT and
+ * UINT4.
+ */
+
+ LONG,
+
+ /**
+ * Type is set from a double: FLOAT4 and FLOAT8.
+ */
+ DOUBLE,
+
+ /**
+ * The value can be set from a string (for convenience).
+ * VARCHAR and VAR16CHAR.
+ */
+
+ STRING,
+
+ /**
+ * The value is set from a byte buffer. VARCHAR (in production
+ * code), VAR16CHAR, VARBINARY.
+ */
+
+ BYTES,
+
+ /**
+ * The value is set from a BigDecimal: any of Drill's decimal
+ * types.
+ */
+
+ DECIMAL,
+
+ /**
+ * The value is set from a Period. Any of Drill's date/time
+ * types. (Note: there is a known bug in which Drill incorrectly
+ * differentiates between local date/times (those without a timezone)
+ * and absolute date/times (those with a timezone.) Caveat emptor.
+ */
+
+ PERIOD,
+
+ /**
+ * The value has no type. This is typically a dummy writer used
+ * for unprojected columns.
+ */
+
+ NULL
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/impl/VectorPrinter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/impl/VectorPrinter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/impl/VectorPrinter.java
new file mode 100644
index 0000000..45847bc
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/impl/VectorPrinter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.impl;
+
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarCharVector;
+
+import com.google.common.base.Charsets;
+
+/**
+ * Handy tool to visualize string and offset vectors for
+ * debugging.
+ */
+
+public class VectorPrinter {
+
+ public static void printOffsets(UInt4Vector vector, int start, int length) {
+ header(vector, start, length);
+ for (int i = start, j = 0; j < length; i++, j++) {
+ if (j > 0) {
+ System.out.print(" ");
+ }
+ System.out.print(vector.getAccessor().get(i));
+ }
+ System.out.print("], addr = ");
+ System.out.println(vector.getBuffer().addr());
+ }
+
+ public static void printStrings(VarCharVector vector, int start, int length) {
+ printOffsets(vector.getOffsetVector(), start, length + 1);
+ header(vector, start, length);
+ System.out.println();
+ for (int i = start, j = 0; j < length; i++, j++) {
+ System.out.print(" ");
+ System.out.print(i);
+ System.out.print(": \"");
+ System.out.print(stringAt(vector, i));
+ System.out.println("\"");
+ }
+ System.out.println("]");
+ }
+
+ public static void header(ValueVector vector, int start, int length) {
+ System.out.print(vector.getClass());
+ System.out.print(": (");
+ System.out.print(start);
+ System.out.print(" - ");
+ System.out.print(start + length - 1);
+ System.out.print("): [");
+ }
+
+ public static String stringAt(VarCharVector vector, int i) {
+ return new String(vector.getAccessor().get(i), Charsets.UTF_8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/package-info.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/package-info.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/package-info.java
index c90a734..990ab13 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/package-info.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/package-info.java
@@ -161,6 +161,190 @@
* the same interface supported the original mutator-based implementation and
* the revised Netty-based implementation. The benefit, however, is stark;
* the direct-to-Netty version is up to 4x faster (for repeated types).
+ *
+ * <h4>Tuple Model</h4>
+ *
+ * Drill has the idea of row and of a map. (A Drill map is much like a "struct":
+ * every instance of the "map" must have the same columns.) Both are instances
+ * of the relational concept of a "tuple." In relational theory, a tuple is
+ * a collection of values in which each value has a name and a position. The
+ * name is for the user, the position (index) allows efficient implementation.
+ * <p>
+ * Drill is unusual among query and DB engines in that it does not normally
+ * use indexes. The reason is easy to understand. Suppose two files contain
+ * columns a and b. File 1, read by minor fragment 0, contains the columns in
+ * the order (a, b). But, file 2, read by minor fragment 1, contains the columns
+ * in the order (b, a). Drill considers this the same schema. Since column
+ * order can vary, Drill has avoided depending on column order. (But, only
+ * partly; many bugs have cropped up because some parts of the code do
+ * require common ordering.)
+ * <p>
+ * Here we observe that column order varies only across fragments. We have
+ * control of the column order within our own fragment. (We can coerce varying
+ * order into a desired order. If the above two files are read by the same
+ * scan operator, then the first file sets the order at (a, b), and the second
+ * files (b, a) order can be coerced into the (a, b) order.
+ * <p>
+ * Given this insight, the readers and writers here promote position to a
+ * first-class concept. Code can access columns by name (for convenience,
+ * especially in testing) or by position (for efficiency.)
+ * <p>
+ * Further, it is often convenient to fetch a column accessor (reader or
+ * writer) once, then cache it. The design here ensures that such caching works
+ * well. The goal is that, eventually, operators will code-generate references
+ * to cached readers and writers instead of generating code that works directly
+ * with the vectors.
+ *
+ * <h4>Lists and Unions</h4>
+ *
+ * Drill provides a List and a Union type. These types are incomplete, buggy
+ * and ill-supported by Drill's operators. However, they are key to Drill's
+ * JSON-powered, schema-free marketing message. Thus, we must support them
+ * in the reader/writer level even if they are broken and under-used elsewhere
+ * in Drill. (If we did not support them, then the JSON readers could not use
+ * the new model, and we'd have to support both the old and new versions, which
+ * would create a bigger mess than we started with.)
+ * <p>
+ * Drill's other types have a more-or-less simple mapping to the relational
+ * model, allowing simple reader and writer interfaces. But, the Union and List
+ * types are not a good fit and cause a very large amount of complexity in the
+ * reader and writer models.
+ * <p>
+ * A Union is just that: it is a container for a variety of typed vectors. It
+ * is like a "union" in C: it has members for each type, but only one type is
+ * in use at any one time. However, unlike C, the implementation is more like
+ * a C "struct" every vector takes space or every row, even if no value is stored
+ * in that row. That is, a Drill union is as if a naive C programmer used a
+ * "struct" when s/he should have used a union.
+ * <p>
+ * Unions are designed to evolve dynamically as data is read. Suppose we read
+ * the following JSON:<pre></code>
+ * {a: 10} {a: "foo"} {a: null} {a: 12.34}
+ * </code></pre>
+ * Here, we discover the need for an Int type, then a Varchar, then mark a
+ * value as null and finally a Float. The union adds the desired types as we
+ * request them. The writer mimics this behavior, using a listener to do the
+ * needed vector work.
+ * <p>
+ * Further, a union can be null. It carries a types vector that indicates the
+ * type of each row. A zero-value indicates that the union as a whole is null.
+ * In this case, null means no value, is is not, say, a null Int or null
+ * Varchar: it is simply null (as in JSON). Since at most one vector within the union
+ * carries a value, the element vectors must also be nullable. This means
+ * that a union has two null bits: one or the union, the other for the
+ * selected type. It is not clear what Drill semantics are supposed to be. Here
+ * the writers assume that either the whole union is null, or that exactly one
+ * member is non-null. Readers are more paranoid: they assume each member is null
+ * if either the union is null or the member itself is null. (Yes, a bit of a
+ * mess...)
+ * <p>
+ * The current union vector format is highly inefficient.
+ * If the union concept is needed, then it should
+ * be redesigned, perhaps as a variable-width vector in which each entry
+ * consists of a type/value pair. (For variable-width values such as
+ * strings, the implementation would be a triple of (type, length,
+ * value). The API here is designed to abstract away the implementation
+ * and should work equally well for the current "union" implementation and
+ * the possible "variant" implementation. As a result, when changing the
+ * API, avoid introducing methods that assume an implementation.
+ * <p>
+ * Lists add another layer of complexity. A list is, logically, a repeated
+ * union. But, for whatever historical reasons, a List can be other things
+ * as well. First, it can have no type at all: a list of nothing. This likely
+ * has no meaning, but the semantics of the List vector allow it. Second, the
+ * List can be an array of a single type in which each entry can be null.
+ * (Normal Repeated types can have an empty array for a row, but cannot have
+ * a null entry. Lists can have either an empty array or a null array in
+ * order to model the JSON <tt>null</tt> and <tt>[]</tt> cases.)
+ * <p>
+ * When a List has a single type, it stores the backing vector directly within
+ * the List. But, a list can also be a list of unions. In this case, the List
+ * stores a union vector as its backing vector. Here, we now have three ways
+ * to indicate null: the List's bits vector, the type vector in the union, and
+ * the bits vector in each element vector. Again, the writer assumes that
+ * if the List vector is null, the entire value for that row is null. The reader
+ * is again paranoid and handles all three nullable states. (Again, a huge
+ * mess.)
+ * <p>
+ * The readers can provide a nice API for these cases since we know the List
+ * format up front. They can present the list as either a nullable array of
+ * a single type, or as an array of unions.
+ * <p>
+ * Writers have more of a challenge. If we knew that a List was being used as
+ * a list of, say, Nullable Int, we could present the List as an array writer
+ * with Int elements. But, the List allows dynamic type addition, as with unions.
+ * (In the case of the list, it has internal special handling for the single vs.
+ * many type case.)
+ * <p>
+ * To isolate the client from the list representation, it is simpler to always
+ * present a List an array of variants. But, this is awkward in the single-type
+ * case. The solution is to use metadata. If the metadata promises to use only
+ * a single type, the writer can use the nullable array of X format. If the
+ * metadata says to use a union (the default), then the List is presented as
+ * an array of unions, even when the list has 0 or 1 member types. (The
+ * complexity here is excessive: Drill should really redesign this feature to make
+ * it simpler and to better fit the relational model.)
+ *
+ * <h4>Vector Evolution</h4>
+ *
+ * Drill uses value vector classes created during the rush to ship Drill 1.0.
+ * They are not optimal: the key value is that the vectors work.
+ * <p>
+ * The Apache Arrow project created a refined version of the vector classes.
+ * Much talk has occurred about ripping out Drill's implementation to use
+ * Arrow instead.
+ * <p>
+ * However, even Arrow has limits:
+ * <ul>
+ * <li>Like Drill, it uses twice the number of positions in the offset vector
+ * as for the values vector. (Drill allocates power-of-two sizes. The offset
+ * vector has one more entry than values. With a power-of-two number of values,
+ * offsets are rounded to the next power of two.)</li>
+ * <li>Like Drill before this work, Arrow does not manage vector sizes; it
+ * allows vectors to grow without bound, causing the memory problems that this
+ * project seeks to resolve.</li>
+ * <li>Like Drill, Arrow implements unions as a space-inefficient collection
+ * of vectors format.</li>
+ * </ul>
+ * If we learn from the above, we might want to create a Value Vectors 2.0
+ * based on the following concepts:
+ * <ul>
+ * <li>Store vector values as a chain of fixed-size buffers. This avoids
+ * memory fragmentation, makes memory allocation much more efficient, is
+ * easier on the client, and avoids internal fragmentation.</li>
+ * <li>Store offsets as the end value, not the start value. This eliminates
+ * the extra offset position, simplifies indexing, and can save on internal
+ * memory fragmentation.</li>
+ * <li>Store unions using "variant encoding" as described above.</li>
+ * </ul>
+ * Such changes would be a huge project if every operator continued to work
+ * directly with vectors and memory buffers. In fact, the cost would be so
+ * high that these improvements might never be done.
+ * <p>
+ * Therefore, a goal of this reader/writer layer is to isolate the operators
+ * from vector implementation. For this to work, the accessors must be at least
+ * as efficient as direct vector access. (They are now more efficient.)
+ * <p>
+ * Once all operators use this layer, a switch to Arrow, or an evolution toward
+ * Value Vectors 2.0 will be much easier. Simply change the vector format and
+ * update the reader and writer implementations. The rest of the code will
+ * remain unchanged. (Note, to achieve this goal, it is important to carefully
+ * design the accessor API [interfaces] to hide implementation details.)
+ *
+ * <h4>Simpler Reader API</h4>
+ *
+ * A key value of Drill is the ability for users to add custom record readers.
+ * But, at present, doing so is very complex because the developer must know
+ * quite a bit about Drill internals. At this level, they must know how to
+ * allocate vectors, how to write to each kind of vector, how to keep track
+ * of array sizes, how to set the vector and batch row counts, and more. In
+ * general, there is only one right way to do this work. (Though some readers
+ * use the old-style vector writers, others work with direct memory instead
+ * of with vectors, and so on.)
+ * <p>
+ * This layer handles all that work, providing a simple API that encourages
+ * more custom readers because the work to create the readers becomes far
+ * simpler. (Other layers tackle other parts of the problem as well.)
*/
package org.apache.drill.exec.vector.accessor;
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractArrayReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractArrayReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractArrayReader.java
deleted file mode 100644
index 7fb0c9d..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractArrayReader.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor.reader;
-
-import org.apache.drill.exec.vector.UInt4Vector.Accessor;
-import org.apache.drill.exec.vector.accessor.ArrayReader;
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import org.apache.drill.exec.vector.accessor.ObjectReader;
-import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
-import org.apache.drill.exec.vector.accessor.TupleReader;
-import org.apache.drill.exec.vector.complex.RepeatedValueVector;
-
-/**
- * Reader for an array-valued column. This reader provides access to specific
- * array members via an array index. This is an abstract base class;
- * subclasses are generated for each repeated value vector type.
- */
-
-public abstract class AbstractArrayReader implements ArrayReader {
-
- /**
- * Object representation of an array reader.
- */
-
- public static class ArrayObjectReader extends AbstractObjectReader {
-
- private AbstractArrayReader arrayReader;
-
- public ArrayObjectReader(AbstractArrayReader arrayReader) {
- this.arrayReader = arrayReader;
- }
-
- @Override
- public void bindIndex(ColumnReaderIndex index) {
- arrayReader.bindIndex(index);
- }
-
- @Override
- public ObjectType type() {
- return ObjectType.ARRAY;
- }
-
- @Override
- public ArrayReader array() {
- return arrayReader;
- }
-
- @Override
- public ScalarElementReader elements() {
- return arrayReader.elements();
- }
-
- @Override
- public Object getObject() {
- return arrayReader.getObject();
- }
-
- @Override
- public String getAsString() {
- return arrayReader.getAsString();
- }
-
- @Override
- public void reposition() {
- arrayReader.reposition();
- }
- }
-
- public static class BaseElementIndex {
- private final ColumnReaderIndex base;
- protected int startOffset;
- protected int length;
-
- public BaseElementIndex(ColumnReaderIndex base) {
- this.base = base;
- }
-
- public int batchIndex() {
- return base.batchIndex();
- }
-
- public void reset(int startOffset, int length) {
- assert length >= 0;
- assert startOffset >= 0;
- this.startOffset = startOffset;
- this.length = length;
- }
-
- public int size() { return length; }
-
- public int elementIndex(int index) {
- if (index < 0 || length <= index) {
- throw new IndexOutOfBoundsException("Index = " + index + ", length = " + length);
- }
- return startOffset + index;
- }
- }
-
- private final Accessor accessor;
- private final VectorAccessor vectorAccessor;
- protected ColumnReaderIndex baseIndex;
- protected BaseElementIndex elementIndex;
-
- public AbstractArrayReader(RepeatedValueVector vector) {
- accessor = vector.getOffsetVector().getAccessor();
- vectorAccessor = null;
- }
-
- public AbstractArrayReader(VectorAccessor vectorAccessor) {
- accessor = null;
- this.vectorAccessor = vectorAccessor;
- }
-
- public void bindIndex(ColumnReaderIndex index) {
- baseIndex = index;
- if (vectorAccessor != null) {
- vectorAccessor.bind(index);
- }
- }
-
- private Accessor accessor() {
- if (accessor != null) {
- return accessor;
- }
- return ((RepeatedValueVector) (vectorAccessor.vector())).getOffsetVector().getAccessor();
- }
-
- public void reposition() {
- final int index = baseIndex.vectorIndex();
- final Accessor curAccesssor = accessor();
- final int startPosn = curAccesssor.get(index);
- elementIndex.reset(startPosn, curAccesssor.get(index + 1) - startPosn);
- }
-
- @Override
- public int size() { return elementIndex.size(); }
-
- @Override
- public ScalarElementReader elements() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ObjectReader entry(int index) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TupleReader tuple(int index) {
- return entry(index).tuple();
- }
-
- @Override
- public ArrayReader array(int index) {
- return entry(index).array();
- }
-
- @Override
- public ObjectReader entry() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TupleReader tuple() {
- return entry().tuple();
- }
-
- @Override
- public ArrayReader array() {
- return entry().array();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractObjectReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractObjectReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractObjectReader.java
index 59a066e..2a80179 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractObjectReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractObjectReader.java
@@ -17,18 +17,18 @@
*/
package org.apache.drill.exec.vector.accessor.reader;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.accessor.ArrayReader;
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+import org.apache.drill.exec.vector.accessor.ColumnReader;
import org.apache.drill.exec.vector.accessor.ObjectReader;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
+import org.apache.drill.exec.vector.accessor.ObjectType;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.TupleReader;
public abstract class AbstractObjectReader implements ObjectReader {
- public abstract void bindIndex(ColumnReaderIndex index);
-
- public void reposition() { }
+ @Override
+ public ColumnMetadata schema() { return reader().schema(); }
@Override
public ScalarReader scalar() {
@@ -45,8 +45,13 @@ public abstract class AbstractObjectReader implements ObjectReader {
throw new UnsupportedOperationException();
}
+ public abstract ReaderEvents events();
+
+ public abstract ColumnReader reader();
+
@Override
- public ScalarElementReader elements() {
- throw new UnsupportedOperationException();
- }
+ public boolean isNull() { return reader().isNull(); }
+
+ @Override
+ public ObjectType type() { return reader().type(); }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
new file mode 100644
index 0000000..203de23
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import java.math.BigDecimal;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.vector.accessor.ColumnReader;
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+import org.apache.drill.exec.vector.accessor.ObjectType;
+import org.apache.drill.exec.vector.accessor.ScalarReader;
+import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
+import org.apache.drill.exec.vector.accessor.ValueType;
+import org.apache.drill.exec.vector.accessor.impl.AccessorUtilities;
+import org.joda.time.Period;
+
+public abstract class AbstractScalarReader implements ScalarReader, ReaderEvents {
+
+ public static class ScalarObjectReader extends AbstractObjectReader {
+
+ private AbstractScalarReader scalarReader;
+
+ public ScalarObjectReader(AbstractScalarReader scalarReader) {
+ this.scalarReader = scalarReader;
+ }
+
+ @Override
+ public ScalarReader scalar() {
+ return scalarReader;
+ }
+
+ @Override
+ public Object getObject() {
+ return scalarReader.getObject();
+ }
+
+ @Override
+ public String getAsString() {
+ return scalarReader.getAsString();
+ }
+
+ @Override
+ public ReaderEvents events() { return scalarReader; }
+
+ @Override
+ public ColumnReader reader() { return scalarReader; }
+ }
+
+ public static class NullReader extends AbstractScalarReader {
+
+ protected final ColumnMetadata schema;
+
+ protected NullReader(ColumnMetadata schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public ValueType valueType() { return ValueType.NULL; }
+
+ @Override
+ public boolean isNull() { return true; }
+
+ @Override
+ public void bindIndex(ColumnReaderIndex rowIndex) { }
+
+ @Override
+ public ColumnMetadata schema() { return schema; }
+ }
+
+ protected ColumnReaderIndex vectorIndex;
+ protected NullStateReader nullStateReader;
+
+ public static ScalarObjectReader nullReader(ColumnMetadata schema) {
+ return new ScalarObjectReader(new NullReader(schema));
+ }
+
+ @Override
+ public void bindIndex(ColumnReaderIndex rowIndex) {
+ vectorIndex = rowIndex;
+ nullStateReader.bindIndex(rowIndex);
+ }
+
+ @Override
+ public void bindNullState(NullStateReader nullStateReader) {
+ this.nullStateReader = nullStateReader;
+ }
+
+ @Override
+ public ObjectType type() { return ObjectType.SCALAR; }
+
+ @Override
+ public NullStateReader nullStateReader() { return nullStateReader; }
+
+ @Override
+ public void reposition() { }
+
+ @Override
+ public boolean isNull() {
+ return nullStateReader.isNull();
+ }
+
+ protected UnsupportedConversionError conversionError(String javaType) {
+ return UnsupportedConversionError.writeError(schema(), javaType);
+ }
+
+ @Override
+ public int getInt() {
+ throw conversionError("int");
+ }
+
+ @Override
+ public long getLong() {
+ throw conversionError("long");
+ }
+
+ @Override
+ public double getDouble() {
+ throw conversionError("double");
+ }
+
+ @Override
+ public String getString() {
+ throw conversionError("String");
+ }
+
+ @Override
+ public byte[] getBytes() {
+ throw conversionError("bytes");
+ }
+
+ @Override
+ public BigDecimal getDecimal() {
+ throw conversionError("Decimal");
+ }
+
+ @Override
+ public Period getPeriod() {
+ throw conversionError("Period");
+ }
+
+ @Override
+ public Object getObject() {
+ if (isNull()) {
+ return null;
+ }
+ switch (valueType()) {
+ case BYTES:
+ return getBytes();
+ case DECIMAL:
+ return getDecimal();
+ case DOUBLE:
+ return getDouble();
+ case INTEGER:
+ return getInt();
+ case LONG:
+ return getLong();
+ case PERIOD:
+ return getPeriod();
+ case STRING:
+ return getString();
+ default:
+ throw new IllegalStateException("Unexpected type: " + valueType());
+ }
+ }
+
+ @Override
+ public String getAsString() {
+ if (isNull()) {
+ return "null";
+ }
+ switch (valueType()) {
+ case BYTES:
+ return AccessorUtilities.bytesToString(getBytes());
+ case DOUBLE:
+ return Double.toString(getDouble());
+ case INTEGER:
+ return Integer.toString(getInt());
+ case LONG:
+ return Long.toString(getLong());
+ case STRING:
+ return "\"" + getString() + "\"";
+ case DECIMAL:
+ return getDecimal().toPlainString();
+ case PERIOD:
+ return getPeriod().normalizedStandard().toString();
+ default:
+ throw new IllegalArgumentException("Unsupported type " + valueType());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractTupleReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractTupleReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractTupleReader.java
index 0429f3e..2c09e5b 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractTupleReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractTupleReader.java
@@ -20,12 +20,11 @@ package org.apache.drill.exec.vector.accessor.reader;
import java.util.ArrayList;
import java.util.List;
-import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.vector.accessor.ArrayReader;
+import org.apache.drill.exec.vector.accessor.ColumnReader;
import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
import org.apache.drill.exec.vector.accessor.ObjectReader;
import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.TupleReader;
@@ -34,26 +33,15 @@ import org.apache.drill.exec.vector.accessor.TupleReader;
* column using either a name or a numeric index.
*/
-public abstract class AbstractTupleReader implements TupleReader {
+public abstract class AbstractTupleReader implements TupleReader, ReaderEvents {
public static class TupleObjectReader extends AbstractObjectReader {
- private AbstractTupleReader tupleReader;
+ private final AbstractTupleReader tupleReader;
public TupleObjectReader(AbstractTupleReader tupleReader) {
this.tupleReader = tupleReader;
}
-
- @Override
- public void bindIndex(ColumnReaderIndex index) {
- tupleReader.bindIndex(index);
- }
-
- @Override
- public ObjectType type() {
- return ObjectType.TUPLE;
- }
-
@Override
public TupleReader tuple() {
return tupleReader;
@@ -70,30 +58,42 @@ public abstract class AbstractTupleReader implements TupleReader {
}
@Override
- public void reposition() {
- tupleReader.reposition();
- }
+ public ReaderEvents events() { return tupleReader; }
+
+ @Override
+ public ColumnReader reader() { return tupleReader; }
}
- protected final TupleMetadata schema;
private final AbstractObjectReader readers[];
+ protected NullStateReader nullStateReader;
- protected AbstractTupleReader(TupleMetadata schema, AbstractObjectReader readers[]) {
- this.schema = schema;
+ protected AbstractTupleReader(AbstractObjectReader readers[]) {
this.readers = readers;
}
+ @Override
+ public ObjectType type() { return ObjectType.TUPLE; }
+
+ @Override
public void bindIndex(ColumnReaderIndex index) {
for (int i = 0; i < readers.length; i++) {
- readers[i].bindIndex(index);
+ readers[i].events().bindIndex(index);
}
}
@Override
- public TupleMetadata schema() { return schema; }
+ public void bindNullState(NullStateReader nullStateReader) {
+ this.nullStateReader = nullStateReader;
+ }
+
+ @Override
+ public NullStateReader nullStateReader() { return nullStateReader; }
@Override
- public int columnCount() { return schema().size(); }
+ public boolean isNull() { return nullStateReader.isNull(); }
+
+ @Override
+ public int columnCount() { return tupleSchema().size(); }
@Override
public ObjectReader column(int colIndex) {
@@ -102,13 +102,23 @@ public abstract class AbstractTupleReader implements TupleReader {
@Override
public ObjectReader column(String colName) {
- int index = schema.index(colName);
+ int index = tupleSchema().index(colName);
if (index == -1) {
return null; }
return readers[index];
}
@Override
+ public ObjectType type(int colIndex) {
+ return column(colIndex).type();
+ }
+
+ @Override
+ public ObjectType type(String colName) {
+ return column(colName).type();
+ }
+
+ @Override
public ScalarReader scalar(int colIndex) {
return column(colIndex).scalar();
}
@@ -139,28 +149,9 @@ public abstract class AbstractTupleReader implements TupleReader {
}
@Override
- public ObjectType type(int colIndex) {
- return column(colIndex).type();
- }
-
- @Override
- public ObjectType type(String colName) {
- return column(colName).type();
- }
-
- @Override
- public ScalarElementReader elements(int colIndex) {
- return column(colIndex).elements();
- }
-
- @Override
- public ScalarElementReader elements(String colName) {
- return column(colName).elements();
- }
-
public void reposition() {
for (int i = 0; i < columnCount(); i++) {
- readers[i].reposition();
+ readers[i].events().reposition();
}
}
@@ -176,14 +167,14 @@ public abstract class AbstractTupleReader implements TupleReader {
@Override
public String getAsString() {
StringBuilder buf = new StringBuilder();
- buf.append("(");
+ buf.append("{");
for (int i = 0; i < columnCount(); i++) {
if (i > 0) {
buf.append( ", " );
}
buf.append(readers[i].getAsString());
}
- buf.append(")");
+ buf.append("}");
return buf.toString();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ArrayReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ArrayReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ArrayReaderImpl.java
new file mode 100644
index 0000000..7f2bf39
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ArrayReaderImpl.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.vector.accessor.ArrayReader;
+import org.apache.drill.exec.vector.accessor.ColumnReader;
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+import org.apache.drill.exec.vector.accessor.ObjectReader;
+import org.apache.drill.exec.vector.accessor.ObjectType;
+import org.apache.drill.exec.vector.accessor.ScalarReader;
+import org.apache.drill.exec.vector.accessor.TupleReader;
+
+/**
+ * Reader for an array-valued column. This reader provides access to specific
+ * array members via an array index. This class implements all arrays. The
+ * behavior for specific array types (scalar, map, lists, etc.) is provided
+ * through composition.
+ */
+
+public class ArrayReaderImpl implements ArrayReader, ReaderEvents {
+
+ /**
+ * Object representation of an array reader.
+ */
+
+ public static class ArrayObjectReader extends AbstractObjectReader {
+
+ private ArrayReaderImpl arrayReader;
+
+ public ArrayObjectReader(ArrayReaderImpl arrayReader) {
+ this.arrayReader = arrayReader;
+ }
+
+ @Override
+ public ArrayReader array() {
+ return arrayReader;
+ }
+
+ @Override
+ public Object getObject() {
+ return arrayReader.getObject();
+ }
+
+ @Override
+ public String getAsString() {
+ return arrayReader.getAsString();
+ }
+
+ @Override
+ public ReaderEvents events() { return arrayReader; }
+
+ @Override
+ public ColumnReader reader() { return arrayReader; }
+ }
+
+ /**
+ * Index into the vector of elements for a repeated vector.
+ * Indexes elements relative to the array. That is, if an array
+ * has five elements, the index here tracks elements 0..4.
+ * The actual vector index is given as the start offset plus the
+ * offset into the array.
+ * <p>
+ * Indexes allow random or sequential access. Random access is more
+ * typical for scalar arrays, while sequential access can be more convenient
+ * for tuple arrays.
+ */
+
+ public static class ElementReaderIndex implements ColumnReaderIndex {
+ protected final ColumnReaderIndex base;
+ protected int startOffset;
+ protected int length;
+ protected int position;
+
+ public ElementReaderIndex(ColumnReaderIndex base) {
+ this.base = base;
+ }
+
+ @Override
+ public int hyperVectorIndex() { return 0; }
+
+ /**
+ * Reposition this array index for a new array given the array start
+ * offset and length.
+ *
+ * @param startOffset first location within the array's
+ * offset vector
+ * @param length number of offset vector locations associated with this
+ * array
+ */
+
+ public void reset(int startOffset, int length) {
+ assert length >= 0;
+ assert startOffset >= 0;
+ this.startOffset = startOffset;
+ this.length = length;
+ position = -1;
+ }
+
+ public void rewind() {
+ position = -1;
+ }
+
+ @Override
+ public int size() { return length; }
+
+ /**
+ * Given a 0-based index relative to the current array, return an absolute offset
+ * vector location for the array value.
+ *
+ * @param index 0-based index into the current array
+ * @return absolute offset vector location for the array value
+ */
+
+ @Override
+ public int offset() {
+ if (position < 0 || length <= position) {
+ throw new IndexOutOfBoundsException("Index = " + position + ", length = " + length);
+ }
+ return startOffset + position;
+ }
+
+ @Override
+ public boolean next() {
+ if (++position < length) {
+ return true;
+ }
+ position = length;
+ return false;
+ }
+
+ /**
+ * Set the current iterator location to the given index offset.
+ *
+ * @param index 0-based index into the current array
+ */
+
+ public void set(int index) {
+ if (index < 0 || length < index) {
+ throw new IndexOutOfBoundsException("Index = " + index + ", length = " + length);
+ }
+ position = index;
+ }
+
+ @Override
+ public int logicalIndex() { return position; }
+ }
+
+ private final ColumnMetadata schema;
+ private final VectorAccessor arrayAccessor;
+ private final OffsetVectorReader offsetReader;
+ private final AbstractObjectReader elementReader;
+ protected ElementReaderIndex elementIndex;
+ protected NullStateReader nullStateReader;
+
+ public ArrayReaderImpl(ColumnMetadata schema, VectorAccessor va,
+ AbstractObjectReader elementReader) {
+ this.schema = schema;
+ arrayAccessor = va;
+ this.elementReader = elementReader;
+ offsetReader = new OffsetVectorReader(
+ VectorAccessors.arrayOffsetVectorAccessor(va));
+ }
+
+ /**
+ * Build a scalar array for a Repeated type. Such arrays are not nullable.
+ *
+ * @param arrayAccessor vector accessor for the repeated vector that holds
+ * the scalar values
+ * @param elementReader scalar reader used to decode each scalar value
+ * @return object reader which wraps the scalar array reader
+ */
+
+ public static ArrayObjectReader buildScalar(ColumnMetadata schema,
+ VectorAccessor arrayAccessor,
+ BaseScalarReader elementReader) {
+
+ // Reader is bound to the values vector inside the nullable vector.
+
+ elementReader.bindVector(schema,
+ VectorAccessors.arrayDataAccessor(arrayAccessor));
+
+ // The scalar array element can't be null.
+
+ elementReader.bindNullState(NullStateReaders.REQUIRED_STATE_READER);
+
+ // Create the array, giving it an offset vector reader based on the
+ // repeated vector's offset vector.
+
+ ArrayReaderImpl arrayReader = new ArrayReaderImpl(schema, arrayAccessor,
+ new AbstractScalarReader.ScalarObjectReader(elementReader));
+
+ // The array itself can't be null.
+
+ arrayReader.bindNullState(NullStateReaders.REQUIRED_STATE_READER);
+
+ // Wrap it all in an object reader.
+
+ return new ArrayObjectReader(arrayReader);
+ }
+
+ /**
+ * Build a repeated map reader.
+ *
+ * @param arrayAccessor vector accessor for the repeated map vector
+ * @param elementReader tuple reader that provides access to each
+ * tuple in the array
+ * @return object reader that wraps the map array reader
+ */
+
+ public static AbstractObjectReader buildTuple(ColumnMetadata schema,
+ VectorAccessor arrayAccessor,
+ AbstractObjectReader elementReader) {
+
+ // Create the array reader over the map vector.
+
+ ArrayReaderImpl arrayReader = new ArrayReaderImpl(schema, arrayAccessor, elementReader);
+
+ // The array itself can't be null.
+
+ arrayReader.bindNullState(NullStateReaders.REQUIRED_STATE_READER);
+
+ // Wrap it all in an object reader.
+
+ return new ArrayObjectReader(arrayReader);
+ }
+
+ @Override
+ public void bindIndex(ColumnReaderIndex index) {
+ arrayAccessor.bind(index);
+ offsetReader.bindIndex(index);
+ nullStateReader.bindIndex(index);
+ elementIndex = new ElementReaderIndex(index);
+ elementReader.events().bindIndex(elementIndex);
+ }
+
+ @Override
+ public void bindNullState(NullStateReader nullStateReader) {
+ this.nullStateReader = nullStateReader;
+ }
+
+ @Override
+ public ObjectType type() { return ObjectType.ARRAY; }
+
+ @Override
+ public ColumnMetadata schema() { return schema; }
+
+ @Override
+ public NullStateReader nullStateReader() { return nullStateReader; }
+
+ @Override
+ public boolean isNull() { return nullStateReader.isNull(); }
+
+ @Override
+ public void reposition() {
+ long entry = offsetReader.getEntry();
+ elementIndex.reset((int) (entry >> 32), (int) (entry & 0xFFFF_FFFF));
+ }
+
+ @Override
+ public boolean next() {
+ if (! elementIndex.next()) {
+ return false;
+ }
+ elementReader.events().reposition();
+ return true;
+ }
+
+ public ColumnReaderIndex elementIndex() { return elementIndex; }
+
+ @Override
+ public int size() { return elementIndex.size(); }
+
+ @Override
+ public void setPosn(int posn) {
+ elementIndex.set(posn);
+ elementReader.events().reposition();
+ }
+
+ @Override
+ public void rewind() {
+ elementIndex.rewind();
+ }
+
+ @Override
+ public ObjectReader entry() { return elementReader; }
+
+ @Override
+ public ObjectType entryType() { return elementReader.type(); }
+
+ @Override
+ public ScalarReader scalar() {
+ return entry().scalar();
+ }
+
+ @Override
+ public TupleReader tuple() {
+ return entry().tuple();
+ }
+
+ @Override
+ public ArrayReader array() {
+ return entry().array();
+ }
+
+ @Override
+ public Object getObject() {
+
+ // Simple: return elements as an object list.
+ // If really needed, could return as a typed array, but that
+ // is a bit of a hassle.
+
+ rewind();
+ List<Object> elements = new ArrayList<>();
+ while (next()) {
+ elements.add(elementReader.getObject());
+ }
+ return elements;
+ }
+
+ @Override
+ public String getAsString() {
+ if (isNull()) {
+ return "null";
+ }
+ rewind();
+ StringBuilder buf = new StringBuilder();
+ buf.append("[");
+ int i = 0;
+ while (next()) {
+ if (i++ > 0) {
+ buf.append( ", " );
+ }
+ buf.append(elementReader.getAsString());
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+}