You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2018/04/07 12:35:54 UTC
[08/11] drill git commit: DRILL-6230: Extend row set readers to
handle hyper vectors
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
index 352ab34..c402466 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
@@ -433,7 +433,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest {
assertEquals(5, schema.size());
assertEquals(4, schema.index("e"));
assertEquals(4, schema.index("E"));
- rootWriter.array(4).set("e1", "e2", "e3");
+ rootWriter.array(4).setObject(strArray("e1", "e2", "e3"));
rootWriter.save();
// Verify. No reason to expect problems, but might as well check.
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
index cc2a83e..07100ed 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java
@@ -29,7 +29,6 @@ import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.accessor.ArrayReader;
import org.apache.drill.exec.vector.accessor.ArrayWriter;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.TupleReader;
@@ -256,7 +255,8 @@ public class TestResultSetLoaderTorture extends SubOperatorTest {
ScalarReader n2Reader;
ScalarReader s2Reader;
ScalarReader n3Reader;
- ScalarElementReader s3Reader;
+ ArrayReader s3Array;
+ ScalarReader s3Reader;
ReadState readState;
public BatchReader(TestSetup setup, RowSetReader reader, ReadState readState) {
@@ -272,7 +272,8 @@ public class TestResultSetLoaderTorture extends SubOperatorTest {
s2Reader = m2Reader.scalar("s2");
TupleReader m3Reader = m2Reader.tuple("m3");
n3Reader = m3Reader.scalar("n3");
- s3Reader = m3Reader.array("s3").elements();
+ s3Array = m3Reader.array("s3");
+ s3Reader = s3Array.scalar();
}
public void verify() {
@@ -312,7 +313,7 @@ public class TestResultSetLoaderTorture extends SubOperatorTest {
private void verifyM2Array() {
for (int i = 0; i < setup.m2Count; i++) {
- a2Reader.setPosn(i);
+ assert(a2Reader.next());
// n2: usual int
@@ -321,7 +322,7 @@ public class TestResultSetLoaderTorture extends SubOperatorTest {
if (readState.innerCount % setup.s2Cycle == 0) {
// Skipped values should be null
assertTrue(
- String.format("Row %d, entry %d", rootReader.rowIndex(), i),
+ String.format("Row %d, entry %d", rootReader.offset(), i),
s2Reader.isNull());
} else if (readState.innerCount % setup.s2Cycle % setup.nullCycle == 0) {
assertTrue(s2Reader.isNull());
@@ -338,10 +339,11 @@ public class TestResultSetLoaderTorture extends SubOperatorTest {
// s3: a repeated VarChar
if (readState.innerCount % setup.s3Cycle == 0) {
- assertEquals(0, s3Reader.size());
+ assertEquals(0, s3Array.size());
} else {
for (int j = 0; j < setup.s3Count; j++) {
- assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count + j), s3Reader.getString(j));
+ assertTrue(s3Array.next());
+ assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count + j), s3Reader.getString());
}
}
readState.innerCount++;
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
index a511a0a..0882a1c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java
@@ -28,6 +28,7 @@ import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
import org.apache.drill.test.rowSet.schema.SchemaBuilder;
import org.apache.drill.test.rowSet.RowSetComparison;
+import org.apache.drill.test.rowSet.schema.SchemaBuilder;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -72,7 +73,7 @@ public class TestVectorContainer extends DrillTest {
.addRow(30, "wilma")
.build();
- // Simulated "implicit" coumns: row number and file name
+ // Simulated "implicit" columns: row number and file name
BatchSchema rightSchema = new SchemaBuilder()
.add("x", MinorType.SMALLINT)
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java
index 9262706..0e63c0e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/DirectRowSet.java
@@ -20,17 +20,17 @@ package org.apache.drill.test.rowSet;
import com.google.common.collect.Sets;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval;
-import org.apache.drill.exec.physical.rowSet.model.ReaderIndex;
-import org.apache.drill.exec.physical.rowSet.model.SchemaInference;
import org.apache.drill.exec.physical.rowSet.model.single.BaseWriterBuilder;
import org.apache.drill.exec.physical.rowSet.model.single.BuildVectorsFromMetadata;
+import org.apache.drill.exec.physical.rowSet.model.single.DirectRowIndex;
+import org.apache.drill.exec.physical.rowSet.model.single.SingleSchemaInference;
import org.apache.drill.exec.physical.rowSet.model.single.VectorAllocator;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.apache.drill.exec.record.VectorAccessible;
-import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.metadata.MetadataUtils;
import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
import org.apache.drill.test.rowSet.RowSetWriterImpl.WriterIndexImpl;
@@ -44,26 +44,6 @@ import java.util.Set;
public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowSet {
- /**
- * Reader index that points directly to each row in the row set.
- * This index starts with pointing to the -1st row, so that the
- * reader can require a <tt>next()</tt> for every row, including
- * the first. (This is the JDBC RecordSet convention.)
- */
-
- private static class DirectRowIndex extends ReaderIndex {
-
- public DirectRowIndex(int rowCount) {
- super(rowCount);
- }
-
- @Override
- public int vectorIndex() { return rowIndex; }
-
- @Override
- public int batchIndex() { return 0; }
- }
-
public static class RowSetWriterBuilder extends BaseWriterBuilder {
public RowSetWriter buildWriter(DirectRowSet rowSet) {
@@ -94,7 +74,7 @@ public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowS
}
public static DirectRowSet fromContainer(VectorContainer container) {
- return new DirectRowSet(container, new SchemaInference().infer(container));
+ return new DirectRowSet(container, new SingleSchemaInference().infer(container));
}
public static DirectRowSet fromVectorAccessible(BufferAllocator allocator, VectorAccessible va) {
@@ -151,7 +131,6 @@ public class DirectRowSet extends AbstractSingleRowSet implements ExtendableRowS
return new IndirectRowSet(this, skipIndices);
}
-
@Override
public SelectionVector2 getSv2() { return null; }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java
index 70b8a20..542421e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/HyperRowSetImpl.java
@@ -17,11 +17,16 @@
*/
package org.apache.drill.test.rowSet;
-import org.apache.drill.exec.physical.rowSet.model.MetadataProvider.MetadataRetrieval;
-import org.apache.drill.exec.physical.rowSet.model.SchemaInference;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.rowSet.model.hyper.BaseReaderBuilder;
+import org.apache.drill.exec.physical.rowSet.model.hyper.HyperSchemaInference;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.record.ExpandableHyperContainer;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.test.rowSet.RowSet.HyperRowSet;
@@ -45,8 +50,67 @@ public class HyperRowSetImpl extends AbstractRowSet implements HyperRowSet {
TupleMetadata schema = rowSet.schema();
HyperRowIndex rowIndex = new HyperRowIndex(sv4);
return new RowSetReaderImpl(schema, rowIndex,
- buildContainerChildren(rowSet.container(),
- new MetadataRetrieval(schema)));
+ buildContainerChildren(rowSet.container(), schema));
+ }
+ }
+
+ public static class HyperRowSetBuilderImpl implements HyperRowSetBuilder {
+
+ private final BufferAllocator allocator;
+ private final List<VectorContainer> batches = new ArrayList<>();
+ private int totalRowCount;
+
+ public HyperRowSetBuilderImpl(BufferAllocator allocator) {
+ this.allocator = allocator;
+ }
+
+ @Override
+ public void addBatch(SingleRowSet rowSet) {
+ if (rowSet.rowCount() == 0) {
+ return;
+ }
+ if (rowSet.indirectionType() != SelectionVectorMode.NONE) {
+ throw new IllegalArgumentException("Batches must not have a selection vector.");
+ }
+ batches.add(rowSet.container());
+ totalRowCount += rowSet.rowCount();
+ }
+
+ @Override
+ public void addBatch(VectorContainer container) {
+ if (container.getRecordCount() == 0) {
+ return;
+ }
+ if (container.getSchema().getSelectionVectorMode() != SelectionVectorMode.NONE) {
+ throw new IllegalArgumentException("Batches must not have a selection vector.");
+ }
+ batches.add(container);
+ totalRowCount += container.getRecordCount();
+ }
+
+ @SuppressWarnings("resource")
+ @Override
+ public HyperRowSet build() throws SchemaChangeException {
+ SelectionVector4 sv4 = new SelectionVector4(allocator, totalRowCount);
+ ExpandableHyperContainer hyperContainer = new ExpandableHyperContainer();
+ for (VectorContainer container : batches) {
+ hyperContainer.addBatch(container);
+ }
+
+ // TODO: This has a bug. If the hyperset has two batches with unions,
+ // and the first union contains only VARCHAR, while the second contains
+ // only INT, the combined schema should be (VARCHAR, INT). Same is true
+ // of lists. But, this code looks at only the first container.
+ //
+ // This is only a theoretical bug as Drill does not support unions
+ // completely, but must be fixed if we want complete union support.
+ //
+ // Actually, the problem is more fundamental. The extendable hyper
+ // container, which creates the metadata schema, does not handle the
+ // case either.
+
+ TupleMetadata schema = new HyperSchemaInference().infer(hyperContainer);
+ return new HyperRowSetImpl(schema, hyperContainer, sv4);
}
}
@@ -56,13 +120,37 @@ public class HyperRowSetImpl extends AbstractRowSet implements HyperRowSet {
private final SelectionVector4 sv4;
- public HyperRowSetImpl(VectorContainer container, SelectionVector4 sv4) {
- super(container, new SchemaInference().infer(container));
+ public HyperRowSetImpl(TupleMetadata schema, VectorContainer container, SelectionVector4 sv4) {
+ super(container, schema);
this.sv4 = sv4;
}
+ public HyperRowSetImpl(VectorContainer container, SelectionVector4 sv4) throws SchemaChangeException {
+ this(new HyperSchemaInference().infer(container), container, sv4);
+ }
+
+ public static HyperRowSetBuilder builder(BufferAllocator allocator) {
+ return new HyperRowSetBuilderImpl(allocator);
+ }
+
public static HyperRowSet fromContainer(VectorContainer container, SelectionVector4 sv4) {
- return new HyperRowSetImpl(container, sv4);
+ try {
+ return new HyperRowSetImpl(container, sv4);
+ } catch (SchemaChangeException e) {
+ throw new UnsupportedOperationException(e);
+ }
+ }
+
+ public static HyperRowSet fromRowSets(BufferAllocator allocator, SingleRowSet...rowSets) {
+ HyperRowSetBuilder builder = builder(allocator);
+ for (SingleRowSet rowSet : rowSets) {
+ builder.addBatch(rowSet);
+ }
+ try {
+ return builder.build();
+ } catch (SchemaChangeException e) {
+ throw new IllegalArgumentException("Incompatible schemas", e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java
index 42d97db..a2bc5e8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/IndirectRowSet.java
@@ -20,10 +20,10 @@ package org.apache.drill.test.rowSet;
import com.google.common.collect.Sets;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.RecordBatchSizer;
import org.apache.drill.exec.physical.rowSet.model.ReaderIndex;
-import org.apache.drill.exec.physical.rowSet.model.SchemaInference;
+import org.apache.drill.exec.physical.rowSet.model.single.SingleSchemaInference;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.RecordBatchSizer;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.selection.SelectionVector2;
@@ -38,8 +38,8 @@ public class IndirectRowSet extends AbstractSingleRowSet {
/**
* Reader index that points to each row indirectly through the
- * selection vector. The {@link #vectorIndex()} method points to the
- * actual data row, while the {@link #position()} method gives
+ * selection vector. The {@link #physicalIndex()} method points to the
+ * actual data row, while the {@link #logicalIndex()} method gives
* the position relative to the indirection vector. That is,
* the position increases monotonically, but the index jumps
* around as specified by the indirection vector.
@@ -55,16 +55,16 @@ public class IndirectRowSet extends AbstractSingleRowSet {
}
@Override
- public int vectorIndex() { return sv2.getIndex(rowIndex); }
+ public int offset() { return sv2.getIndex(position); }
@Override
- public int batchIndex() { return 0; }
+ public int hyperVectorIndex() { return 0; }
}
private final SelectionVector2 sv2;
private IndirectRowSet(VectorContainer container, SelectionVector2 sv2) {
- super(container, new SchemaInference().infer(container));
+ super(container, new SingleSchemaInference().infer(container));
this.sv2 = sv2;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
index 53be75d..63a4959 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.test.rowSet;
+import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -143,4 +144,10 @@ public interface RowSet {
interface HyperRowSet extends RowSet {
SelectionVector4 getSv4();
}
+
+ interface HyperRowSetBuilder {
+ void addBatch(SingleRowSet rowSet);
+ void addBatch(VectorContainer container);
+ HyperRowSet build() throws SchemaChangeException;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
index 1cae64f..e098e33 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
@@ -20,15 +20,14 @@ package org.apache.drill.test.rowSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.Comparator;
+
import org.apache.drill.exec.vector.accessor.ArrayReader;
import org.apache.drill.exec.vector.accessor.ObjectReader;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.TupleReader;
import org.bouncycastle.util.Arrays;
-import java.util.Comparator;
-
/**
* For testing, compare the contents of two row sets (record batches)
* to verify that they are identical. Supports masks to exclude certain
@@ -149,6 +148,9 @@ public class RowSetComparison {
*/
public void verify(RowSet actual) {
+ assertTrue("Schemas don't match.\n" +
+ "Expected: " + expected.schema().toString() +
+ "\nActual: " + actual.schema(), expected.schema().isEquivalent(actual.schema()));
int testLength = expected.rowCount() - offset;
if (span > -1) {
testLength = span;
@@ -165,7 +167,7 @@ public class RowSetComparison {
for (int i = 0; i < testLength; i++) {
er.next();
ar.next();
- String label = Integer.toString(er.index() + 1);
+ String label = Integer.toString(er.logicalIndex() + 1);
verifyRow(label, er, ar);
}
}
@@ -276,63 +278,12 @@ public class RowSetComparison {
private void verifyArray(String label, ArrayReader ea,
ArrayReader aa) {
- assertEquals(label, ea.entryType(), aa.entryType());
- assertEquals(label, ea.size(), aa.size());
- switch (ea.entryType()) {
- case ARRAY:
- throw new UnsupportedOperationException();
- case SCALAR:
- verifyScalarArray(label, ea.elements(), aa.elements());
- break;
- case TUPLE:
- verifyTupleArray(label, ea, aa);
- break;
- default:
- throw new IllegalStateException( "Unexpected type: " + ea.entryType());
- }
- }
-
- private void verifyTupleArray(String label, ArrayReader ea, ArrayReader aa) {
- for (int i = 0; i < ea.size(); i++) {
- verifyTuple(label + "[" + i + "]", ea.tuple(i), aa.tuple(i));
- }
- }
-
- private void verifyScalarArray(String colLabel, ScalarElementReader ea,
- ScalarElementReader aa) {
- assertEquals(colLabel, ea.valueType(), aa.valueType());
- assertEquals(colLabel, ea.size(), aa.size());
- for (int i = 0; i < ea.size(); i++) {
- String label = colLabel + "[" + i + "]";
- switch (ea.valueType()) {
- case BYTES: {
- byte expected[] = ea.getBytes(i);
- byte actual[] = aa.getBytes(i);
- assertEquals(label + " - byte lengths differ", expected.length, actual.length);
- assertTrue(label, Arrays.areEqual(expected, actual));
- break;
- }
- case DOUBLE:
- assertEquals(label, ea.getDouble(i), aa.getDouble(i), delta);
- break;
- case INTEGER:
- assertEquals(label, ea.getInt(i), aa.getInt(i));
- break;
- case LONG:
- assertEquals(label, ea.getLong(i), aa.getLong(i));
- break;
- case STRING:
- assertEquals(label, ea.getString(i), aa.getString(i));
- break;
- case DECIMAL:
- assertEquals(label, ea.getDecimal(i), aa.getDecimal(i));
- break;
- case PERIOD:
- assertEquals(label, ea.getPeriod(i), aa.getPeriod(i));
- break;
- default:
- throw new IllegalStateException( "Unexpected type: " + ea.valueType());
- }
+ assertEquals(label + " - array element type", ea.entryType(), aa.entryType());
+ assertEquals(label + " - array length", ea.size(), aa.size());
+ int i = 0;
+ while (ea.next()) {
+ assertTrue(aa.next());
+ verifyColumn(label + "[" + i++ + "]", ea.entry(), aa.entry());
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
index fe50197..82d4bbd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
@@ -41,7 +41,7 @@ public class RowSetPrinter {
public void print(PrintStream out) {
SelectionVectorMode selectionMode = rowSet.indirectionType();
RowSetReader reader = rowSet.reader();
- int colCount = reader.schema().size();
+ int colCount = reader.tupleSchema().size();
printSchema(out, selectionMode, reader);
while (reader.next()) {
printHeader(out, reader, selectionMode);
@@ -68,12 +68,12 @@ public class RowSetPrinter {
break;
}
out.print(": ");
- TupleMetadata schema = reader.schema();
+ TupleMetadata schema = reader.tupleSchema();
printTupleSchema(out, schema);
out.println();
}
- private void printTupleSchema(PrintStream out, TupleMetadata schema) {
+ public static void printTupleSchema(PrintStream out, TupleMetadata schema) {
for (int i = 0; i < schema.size(); i++) {
if (i > 0) {
out.print(", ");
@@ -81,26 +81,26 @@ public class RowSetPrinter {
ColumnMetadata colSchema = schema.metadata(i);
out.print(colSchema.name());
if (colSchema.isMap()) {
- out.print("(");
+ out.print("{");
printTupleSchema(out, colSchema.mapSchema());
- out.print(")");
+ out.print("}");
}
}
}
private void printHeader(PrintStream out, RowSetReader reader, SelectionVectorMode selectionMode) {
- out.print(reader.index());
+ out.print(reader.logicalIndex());
switch (selectionMode) {
case FOUR_BYTE:
out.print(" (");
- out.print(reader.batchIndex());
+ out.print(reader.hyperVectorIndex());
out.print(", ");
- out.print(reader.rowIndex());
+ out.print(reader.offset());
out.print(")");
break;
case TWO_BYTE:
out.print(" (");
- out.print(reader.rowIndex());
+ out.print(reader.offset());
out.print(")");
break;
default:
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java
index 3e27529..4a966cd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReader.java
@@ -32,23 +32,22 @@ public interface RowSetReader extends TupleReader {
int rowCount();
boolean next();
- int index();
- void set(int index);
+ int logicalIndex();
+ void setPosn(int index);
/**
* Batch index: 0 for a single batch, batch for the current
* row is a hyper-batch.
* @return index of the batch for the current row
*/
- int batchIndex();
+ int hyperVectorIndex();
/**
* The index of the underlying row which may be indexed by an
- * Sv2 or Sv4.
+ * SV2 or SV4.
*
* @return
*/
- int rowIndex();
- boolean valid();
+ int offset();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java
index 7217187..6344419 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetReaderImpl.java
@@ -20,9 +20,11 @@ package org.apache.drill.test.rowSet;
import java.util.List;
import org.apache.drill.exec.physical.rowSet.model.ReaderIndex;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.vector.accessor.reader.AbstractObjectReader;
import org.apache.drill.exec.vector.accessor.reader.AbstractTupleReader;
+import org.apache.drill.exec.vector.accessor.reader.NullStateReaders;
/**
* Reader implementation for a row set.
@@ -30,11 +32,14 @@ import org.apache.drill.exec.vector.accessor.reader.AbstractTupleReader;
public class RowSetReaderImpl extends AbstractTupleReader implements RowSetReader {
+ private final TupleMetadata schema;
protected final ReaderIndex readerIndex;
public RowSetReaderImpl(TupleMetadata schema, ReaderIndex index, AbstractObjectReader[] readers) {
- super(schema, readers);
+ super(readers);
+ this.schema = schema;
this.readerIndex = index;
+ bindNullState(NullStateReaders.REQUIRED_STATE_READER);
bindIndex(index);
}
@@ -54,23 +59,26 @@ public class RowSetReaderImpl extends AbstractTupleReader implements RowSetReade
}
@Override
- public boolean valid() { return readerIndex.valid(); }
-
- @Override
- public int index() { return readerIndex.position(); }
+ public int logicalIndex() { return readerIndex.logicalIndex(); }
@Override
public int rowCount() { return readerIndex.size(); }
@Override
- public int rowIndex() { return readerIndex.vectorIndex(); }
+ public int offset() { return readerIndex.offset(); }
@Override
- public int batchIndex() { return readerIndex.batchIndex(); }
+ public int hyperVectorIndex() { return readerIndex.hyperVectorIndex(); }
@Override
- public void set(int index) {
+ public void setPosn(int index) {
this.readerIndex.set(index);
reposition();
}
+
+ @Override
+ public ColumnMetadata schema() { return null; }
+
+ @Override
+ public TupleMetadata tupleSchema() { return schema; }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java
index e47e552..1aa636d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java
@@ -17,9 +17,9 @@
*/
package org.apache.drill.test.rowSet.schema;
-import org.apache.drill.common.types.Types;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.record.metadata.AbstractColumnMetadata;
import org.apache.drill.exec.record.metadata.VariantColumnMetadata;
import org.apache.drill.exec.record.metadata.VariantSchema;
@@ -33,7 +33,6 @@ public class UnionBuilder implements SchemaContainer {
private final SchemaContainer parent;
private final String name;
private final MinorType type;
- private final DataMode mode;
private final VariantSchema union;
public UnionBuilder(SchemaContainer parent, String name,
@@ -41,7 +40,6 @@ public class UnionBuilder implements SchemaContainer {
this.parent = parent;
this.name = name;
this.type = type;
- this.mode = mode;
union = new VariantSchema();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
index 9ad4133..92b9266 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
@@ -36,7 +36,6 @@ import org.apache.drill.exec.vector.VectorOverflowException;
import org.apache.drill.exec.vector.accessor.ArrayReader;
import org.apache.drill.exec.vector.accessor.ArrayWriter;
import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.TupleReader;
@@ -45,6 +44,7 @@ import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.RepeatedMapVector;
import org.apache.drill.test.SubOperatorTest;
+import org.apache.drill.test.rowSet.DirectRowSet;
import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
import org.apache.drill.test.rowSet.RowSetComparison;
@@ -60,12 +60,17 @@ import org.junit.Test;
* Tests basic protocol of the writers: <pre><code>
* row : tuple
* tuple : column *
- * column : scalar obj | array obj | tuple obj
+ * column : scalar obj | array obj | tuple obj | variant obj
* scalar obj : scalar
- * array obj : array writer
- * array writer : element
+ * array obj : array
+ * array : index --> column
* element : column
- * tuple obj : tuple</code></pre>
+ * tuple obj : tuple
+ * tuple : name --> column (also index --> column)
+ * variant obj : variant
+ * variant : type --> column</code></pre>
+ * <p>
+ * A list is an array of variants. Variants are tested elsewhere.
*/
public class RowSetTest extends SubOperatorTest {
@@ -137,16 +142,21 @@ public class RowSetTest extends SubOperatorTest {
assertSame(reader.column("a").scalar(), reader.scalar("a"));
assertSame(reader.column(0).scalar(), reader.scalar(0));
assertEquals(ValueType.INTEGER, reader.scalar(0).valueType());
+ assertTrue(schema.metadata("a").isEquivalent(reader.column("a").schema()));
// Test various accessors: full and simple
assertTrue(reader.next());
+ assertFalse(reader.column("a").scalar().isNull());
assertEquals(10, reader.column("a").scalar().getInt());
assertTrue(reader.next());
+ assertFalse(reader.scalar("a").isNull());
assertEquals(20, reader.scalar("a").getInt());
assertTrue(reader.next());
+ assertFalse(reader.column(0).scalar().isNull());
assertEquals(30, reader.column(0).scalar().getInt());
assertTrue(reader.next());
+ assertFalse(reader.column(0).scalar().isNull());
assertEquals(40, reader.scalar(0).getInt());
assertFalse(reader.next());
@@ -238,27 +248,50 @@ public class RowSetTest extends SubOperatorTest {
assertSame(reader.column(0).array(), reader.array(0));
assertEquals(ObjectType.SCALAR, reader.column("a").array().entryType());
- assertEquals(ValueType.INTEGER, reader.array(0).elements().valueType());
+ assertEquals(ValueType.INTEGER, reader.array(0).scalar().valueType());
// Read and verify the rows
- ScalarElementReader intReader = reader.array(0).elements();
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader intReader = arrayReader.scalar();
assertTrue(reader.next());
- assertEquals(2, intReader.size());
- assertEquals(10, intReader.getInt(0));
- assertEquals(11, intReader.getInt(1));
+ assertFalse(arrayReader.isNull());
+ assertEquals(2, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(10, intReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(11, intReader.getInt());
+ assertFalse(arrayReader.next());
+
assertTrue(reader.next());
- assertEquals(3, intReader.size());
- assertEquals(20, intReader.getInt(0));
- assertEquals(21, intReader.getInt(1));
- assertEquals(22, intReader.getInt(2));
+ assertFalse(arrayReader.isNull());
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(20, intReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(21, intReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(22, intReader.getInt());
+ assertFalse(arrayReader.next());
+
assertTrue(reader.next());
- assertEquals(1, intReader.size());
- assertEquals(30, intReader.getInt(0));
+ assertFalse(arrayReader.isNull());
+ assertEquals(1, arrayReader.size());
+ assertTrue(arrayReader.next());
+ assertEquals(30, intReader.getInt());
+ assertFalse(arrayReader.next());
+
assertTrue(reader.next());
- assertEquals(2, intReader.size());
- assertEquals(40, intReader.getInt(0));
- assertEquals(41, intReader.getInt(1));
+ assertFalse(arrayReader.isNull());
+ assertEquals(2, arrayReader.size());
+ assertTrue(arrayReader.next());
+ assertEquals(40, intReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(41, intReader.getInt());
+ assertFalse(arrayReader.next());
+
assertFalse(reader.next());
// Test the above again via the writer and reader
@@ -352,22 +385,47 @@ public class RowSetTest extends SubOperatorTest {
ScalarReader aReader = reader.column(0).scalar();
TupleReader mReader = reader.column(1).tuple();
- assertEquals(ObjectType.SCALAR, mReader.column("b").array().entryType());
- ScalarElementReader bReader = mReader.column(0).elements();
+ ArrayReader bArray = mReader.column("b").array();
+ assertEquals(ObjectType.SCALAR, bArray.entryType());
+ ScalarReader bReader = bArray.scalar();
assertEquals(ValueType.INTEGER, bReader.valueType());
+ // Row 1: (10, {[11, 12]})
+
assertTrue(reader.next());
assertEquals(10, aReader.getInt());
- assertEquals(11, bReader.getInt(0));
- assertEquals(12, bReader.getInt(1));
+ assertFalse(mReader.isNull());
+
+ assertTrue(bArray.next());
+ assertFalse(bReader.isNull());
+ assertEquals(11, bReader.getInt());
+ assertTrue(bArray.next());
+ assertFalse(bReader.isNull());
+ assertEquals(12, bReader.getInt());
+ assertFalse(bArray.next());
+
+ // Row 2: (20, {[21, 22]})
+
assertTrue(reader.next());
assertEquals(20, aReader.getInt());
- assertEquals(21, bReader.getInt(0));
- assertEquals(22, bReader.getInt(1));
+ assertFalse(mReader.isNull());
+
+ assertTrue(bArray.next());
+ assertEquals(21, bReader.getInt());
+ assertTrue(bArray.next());
+ assertEquals(22, bReader.getInt());
+
+ // Row 3: (30, {[31, 32]})
+
assertTrue(reader.next());
assertEquals(30, aReader.getInt());
- assertEquals(31, bReader.getInt(0));
- assertEquals(32, bReader.getInt(1));
+ assertFalse(mReader.isNull());
+
+ assertTrue(bArray.next());
+ assertEquals(31, bReader.getInt());
+ assertTrue(bArray.next());
+ assertEquals(32, bReader.getInt());
+
assertFalse(reader.next());
// Verify that the map accessor's value count was set.
@@ -472,18 +530,22 @@ public class RowSetTest extends SubOperatorTest {
assertEquals(ValueType.INTEGER, bReader.valueType());
assertEquals(ValueType.INTEGER, cReader.valueType());
- // Row 1: use index accessors
+ // Row 1: Use iterator-like accessors
assertTrue(reader.next());
assertEquals(10, aReader.getInt());
- TupleReader ixReader = maReader.tuple(0);
- assertEquals(101, ixReader.scalar(0).getInt());
- assertEquals(102, ixReader.scalar(1).getInt());
- ixReader = maReader.tuple(1);
- assertEquals(111, ixReader.scalar(0).getInt());
- assertEquals(112, ixReader.scalar(1).getInt());
-
- // Row 2: use common accessor with explicit positioning,
+ assertFalse(maReader.isNull()); // Array itself is not null
+
+ assertTrue(maReader.next());
+ assertFalse(mapReader.isNull()); // Tuple 0 is not null
+ assertEquals(101, mapReader.scalar(0).getInt());
+ assertEquals(102, mapReader.scalar(1).getInt());
+
+ assertTrue(maReader.next());
+ assertEquals(111, mapReader.scalar(0).getInt());
+ assertEquals(112, mapReader.scalar(1).getInt());
+
+ // Row 2: use explicit positioning,
// but access scalars through the map reader.
assertTrue(reader.next());
@@ -495,14 +557,16 @@ public class RowSetTest extends SubOperatorTest {
assertEquals(211, mapReader.scalar(0).getInt());
assertEquals(212, mapReader.scalar(1).getInt());
- // Row 3: use common accessor for scalars
+ // Row 3: use scalar accessor
assertTrue(reader.next());
assertEquals(30, aReader.getInt());
- maReader.setPosn(0);
+
+ assertTrue(maReader.next());
assertEquals(301, bReader.getInt());
assertEquals(302, cReader.getInt());
- maReader.setPosn(1);
+
+ assertTrue(maReader.next());
assertEquals(311, bReader.getInt());
assertEquals(312, cReader.getInt());
@@ -555,18 +619,28 @@ public class RowSetTest extends SubOperatorTest {
SingleRowSet result = writer.done();
RowSetReader reader = result.reader();
+ ArrayReader arrayReader = reader.array(1);
+ ScalarReader elementReader = arrayReader.scalar();
+
assertTrue(reader.next());
assertEquals(10, reader.scalar(0).getInt());
- ScalarElementReader arrayReader = reader.array(1).elements();
assertEquals(2, arrayReader.size());
- assertEquals(100, arrayReader.getInt(0));
- assertEquals(110, arrayReader.getInt(1));
+
+ assertTrue(arrayReader.next());
+ assertEquals(100, elementReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(110, elementReader.getInt());
+
assertTrue(reader.next());
assertEquals(20, reader.scalar(0).getInt());
assertEquals(3, arrayReader.size());
- assertEquals(200, arrayReader.getInt(0));
- assertEquals(120, arrayReader.getInt(1));
- assertEquals(220, arrayReader.getInt(2));
+ assertTrue(arrayReader.next());
+ assertEquals(200, elementReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(120, elementReader.getInt());
+ assertTrue(arrayReader.next());
+ assertEquals(220, elementReader.getInt());
+
assertTrue(reader.next());
assertEquals(30, reader.scalar(0).getInt());
assertEquals(0, arrayReader.size());
@@ -581,7 +655,6 @@ public class RowSetTest extends SubOperatorTest {
new RowSetComparison(rs1)
.verifyAndClearAll(rs2);
}
-
/**
* Test filling a row set up to the maximum number of rows.
* Values are small enough to prevent filling to the
@@ -666,4 +739,107 @@ public class RowSetTest extends SubOperatorTest {
assertEquals(count, rs.rowCount());
rs.clear();
}
+
+ /**
+ * The code below is not a test. Rather, it is a simple example of
+ * how to write a batch of data using writers, then read it using
+ * readers.
+ */
+
+ @Test
+ public void example() {
+
+ // Step 1: Define a schema. In a real app, this
+ // will be provided by a reader, by an incoming batch,
+ // etc.
+
+ BatchSchema schema = new SchemaBuilder()
+ .add("a", MinorType.VARCHAR)
+ .addArray("b", MinorType.INT)
+ .addMap("c")
+ .add("c1", MinorType.INT)
+ .add("c2", MinorType.VARCHAR)
+ .resumeSchema()
+ .build();
+
+ // Step 2: Create a batch. Done here because this is
+ // a batch-oriented test. Done automatically in the
+ // result set loader.
+
+ DirectRowSet drs = DirectRowSet.fromSchema(fixture.allocator(), schema);
+
+ // Step 3: Create the writer.
+
+ RowSetWriter writer = drs.writer();
+
+ // Step 4: Populate data. Here we do it the way an app would:
+ // using the individual accessors. See tests above for the many
+ // ways this can be done depending on the need of the app.
+ //
+ // Write two rows:
+ // ("fred", [10, 11], {12, "wilma"})
+ // ("barney", [20, 21], {22, "betty"})
+ //
+ // This example uses Java strings for Varchar. Real code might
+ // use byte arrays.
+
+ writer.scalar("a").setString("fred");
+ ArrayWriter bWriter = writer.array("b");
+ bWriter.scalar().setInt(10);
+ bWriter.scalar().setInt(11);
+ TupleWriter cWriter = writer.tuple("c");
+ cWriter.scalar("c1").setInt(12);
+ cWriter.scalar("c2").setString("wilma");
+ writer.save();
+
+ writer.scalar("a").setString("barney");
+ bWriter.scalar().setInt(20);
+ bWriter.scalar().setInt(21);
+ cWriter.scalar("c1").setInt(22);
+ cWriter.scalar("c2").setString("betty");
+ writer.save();
+
+ // Step 5: "Harvest" the batch. Done differently in the
+ // result set loader.
+
+ SingleRowSet rowSet = writer.done();
+
+ // Step 5: Create a reader.
+
+ RowSetReader reader = rowSet.reader();
+
+ // Step 6: Retrieve the data. Here we just print the
+ // values.
+
+ while (reader.next()) {
+ print(reader.scalar("a").getString());
+ ArrayReader bReader = reader.array("b");
+ while (bReader.next()) {
+ print(bReader.scalar().getInt());
+ }
+ TupleReader cReader = reader.tuple("c");
+ print(cReader.scalar("c1").getInt());
+ print(cReader.scalar("c2").getString());
+ endRow();
+ }
+
+ // Step 7: Free memory.
+
+ rowSet.clear();
+ }
+
+ public void print(Object obj) {
+ if (obj instanceof String) {
+ System.out.print("\"");
+ System.out.print(obj);
+ System.out.print("\"");
+ } else {
+ System.out.print(obj);
+ }
+ System.out.print(" ");
+ }
+
+ public void endRow() {
+ System.out.println();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java
index 490a9fd..f4c00ca 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java
@@ -24,17 +24,17 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.record.metadata.TupleMetadata;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
+import org.apache.drill.exec.vector.accessor.ArrayReader;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.schema.SchemaBuilder;
import org.apache.drill.test.rowSet.RowSetReader;
import org.apache.drill.test.rowSet.RowSetUtilities;
import org.apache.drill.test.rowSet.RowSetWriter;
+import org.apache.drill.test.rowSet.schema.SchemaBuilder;
import org.junit.Test;
/**
@@ -219,16 +219,18 @@ public class TestFillEmpties extends SubOperatorTest {
}
SingleRowSet result = writer.done();
RowSetReader reader = result.reader();
- ScalarElementReader colReader = reader.array(0).elements();
+ ArrayReader aReader = reader.array(0);
+ ScalarReader colReader = aReader.scalar();
for (int i = 0; i < ROW_COUNT; i++) {
assertTrue(reader.next());
if (i % 5 != 0) {
// Empty arrays are defined to be the same as a zero-length array.
- assertEquals(0, colReader.size());
+ assertEquals(0, aReader.size());
} else {
for (int j = 0; j < 2; j++) {
- Object actual = colReader.getObject(j);
+ assertTrue(aReader.next());
+ Object actual = colReader.getObject();
Object expected = RowSetUtilities.testDataFromInt(valueType, majorType, i + j);
RowSetUtilities.assertEqualValues(
majorType.toString().replace('\n', ' ') + "[" + i + "][" + j + "]",
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java
new file mode 100644
index 0000000..00e72d4
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test.rowSet.test;
+
+import static org.apache.drill.test.rowSet.RowSetUtilities.mapArray;
+import static org.apache.drill.test.rowSet.RowSetUtilities.mapValue;
+import static org.apache.drill.test.rowSet.RowSetUtilities.strArray;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.test.SubOperatorTest;
+import org.apache.drill.test.rowSet.HyperRowSetImpl;
+import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
+import org.apache.drill.test.rowSet.RowSet.HyperRowSet;
+import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
+import org.apache.drill.test.rowSet.RowSetBuilder;
+import org.apache.drill.test.rowSet.RowSetReader;
+import org.apache.drill.test.rowSet.RowSetUtilities;
+import org.apache.drill.test.rowSet.RowSetWriter;
+import org.apache.drill.test.rowSet.schema.SchemaBuilder;
+import org.junit.Test;
+
+/**
+ * Test the reader mechanism that reads rows indexed via an SV4.
+ * SV4's introduce an additional level of indexing: each row may
+ * come from a different batch. The readers use the SV4 to find
+ * the root batch and vector, then must navigate downward from that
+ * vector for maps, repeated maps, lists, unions, repeated lists,
+ * nullable vectors and variable-length vectors.
+ * <p>
+ * This test does not cover repeated vectors; those tests should be added.
+ */
+
+public class TestHyperVectorReaders extends SubOperatorTest {
+
+ /**
+ * Test the simplest case: a top-level required vector. Has no contained vectors.
+ * This test focuses on the SV4 indirection mechanism itself.
+ */
+
+ @Test
+ public void testRequired() {
+ TupleMetadata schema = new SchemaBuilder()
+ .add("a", MinorType.INT)
+ .buildSchema();
+
+ SingleRowSet rowSet1;
+ {
+ ExtendableRowSet rowSet = fixture.rowSet(schema);
+ RowSetWriter writer = rowSet.writer();
+ for (int i = 0; i < 10; i++) {
+ writer.scalar(0).setInt(i * 10);
+ writer.save();
+ }
+ rowSet1 = writer.done();
+ }
+
+ SingleRowSet rowSet2;
+ {
+ ExtendableRowSet rowSet = fixture.rowSet(schema);
+ RowSetWriter writer = rowSet.writer();
+ for (int i = 10; i < 20; i++) {
+ writer.scalar(0).setInt(i * 10);
+ writer.save();
+ }
+ rowSet2 = writer.done();
+ }
+
+ // Build the hyper batch
+ // [0, 10, 20, ... 190]
+
+ HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2);
+ assertEquals(20, hyperSet.rowCount());
+
+ // Populate the indirection vector:
+ // (1, 9), (0, 9), (1, 8), (0, 8), ... (0, 0)
+
+ SelectionVector4 sv4 = hyperSet.getSv4();
+ for (int i = 0; i < 20; i++) {
+ int batch = i % 2;
+ int offset = 9 - i / 2;
+ sv4.set(i, batch, offset);
+ }
+
+ // Sanity check.
+
+ for (int i = 0; i < 20; i++) {
+ int batch = i % 2;
+ int offset = 9 - i / 2;
+ int encoded = sv4.get(i);
+ assertEquals(batch, SelectionVector4.getBatchIndex(encoded));
+ assertEquals(offset, SelectionVector4.getRecordIndex(encoded));
+ }
+
+ // Verify reader
+ // Expected: [190, 90, 180, 80, ... 0]
+
+ RowSetReader reader = hyperSet.reader();
+ for (int i = 0; i < 20; i++) {
+ assertTrue(reader.next());
+ int batch = i % 2;
+ int offset = 9 - i / 2;
+ int expected = batch * 100 + offset * 10;
+ assertEquals(expected, reader.scalar(0).getInt());
+ }
+ assertFalse(reader.next());
+
+ // Validate using an expected result set.
+
+ RowSetBuilder rsBuilder = fixture.rowSetBuilder(schema);
+ for (int i = 0; i < 20; i++) {
+ int batch = i % 2;
+ int offset = 9 - i / 2;
+ int expected = batch * 100 + offset * 10;
+ rsBuilder.addRow(expected);
+ }
+
+ RowSetUtilities.verify(rsBuilder.build(), hyperSet);
+ }
+
+ @Test
+ public void testVarWidth() {
+ TupleMetadata schema = new SchemaBuilder()
+ .add("a", MinorType.VARCHAR)
+ .buildSchema();
+
+ SingleRowSet rowSet1 = fixture.rowSetBuilder(schema)
+ .addSingleCol("second")
+ .addSingleCol("fourth")
+ .build();
+
+ SingleRowSet rowSet2 = fixture.rowSetBuilder(schema)
+ .addSingleCol("first")
+ .addSingleCol("third")
+ .build();
+
+ // Build the hyper batch
+
+ HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2);
+ assertEquals(4, hyperSet.rowCount());
+ SelectionVector4 sv4 = hyperSet.getSv4();
+ sv4.set(0, 1, 0);
+ sv4.set(1, 0, 0);
+ sv4.set(2, 1, 1);
+ sv4.set(3, 0, 1);
+
+ SingleRowSet expected = fixture.rowSetBuilder(schema)
+ .addRow("first")
+ .addRow("second")
+ .addRow("third")
+ .addRow("fourth")
+ .build();
+
+ RowSetUtilities.verify(expected, hyperSet);
+ }
+
+ /**
+ * Test a nullable varchar. Requires multiple indirections:
+ * <ul>
+ * <li>From the SV4 to the nullable vector.</li>
+ * <li>From the nullable vector to the bits vector.</li>
+ * <li>From the nullable vector to the data vector.</li>
+ * <li>From the data vector to the offset vector.</li>
+ * <li>From the data vector to the values vector.</li>
+ * </ul>
+ * All are coordinated by the vector index and vector accessors.
+ * This test verifies that each of the indirections does, in fact,
+ * work as expected.
+ */
+
+ @Test
+ public void testOptional() {
+ TupleMetadata schema = new SchemaBuilder()
+ .addNullable("a", MinorType.VARCHAR)
+ .buildSchema();
+
+ SingleRowSet rowSet1 = fixture.rowSetBuilder(schema)
+ .addSingleCol("sixth")
+ .addSingleCol(null)
+ .addSingleCol("fourth")
+ .build();
+
+ SingleRowSet rowSet2 = fixture.rowSetBuilder(schema)
+ .addSingleCol(null)
+ .addSingleCol("first")
+ .addSingleCol("third")
+ .build();
+
+ // Build the hyper batch
+
+ HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2);
+ assertEquals(6, hyperSet.rowCount());
+ SelectionVector4 sv4 = hyperSet.getSv4();
+ sv4.set(0, 1, 1);
+ sv4.set(1, 0, 1);
+ sv4.set(2, 1, 2);
+ sv4.set(3, 0, 2);
+ sv4.set(4, 1, 0);
+ sv4.set(5, 0, 0);
+
+ SingleRowSet expected = fixture.rowSetBuilder(schema)
+ .addSingleCol("first")
+ .addSingleCol(null)
+ .addSingleCol("third")
+ .addSingleCol("fourth")
+ .addSingleCol(null)
+ .addSingleCol("sixth")
+ .build();
+
+ RowSetUtilities.verify(expected, hyperSet);
+ }
+
+ /**
+ * Test an array to test the indirection from the repeated vector
+ * to the array offsets vector and the array values vector. (Uses
+ * varchar to add another level of indirection to the data offset
+ * and data values vectors.)
+ */
+
+ @Test
+ public void testRepeated() {
+ TupleMetadata schema = new SchemaBuilder()
+ .addArray("a", MinorType.VARCHAR)
+ .buildSchema();
+
+ SingleRowSet rowSet1 = fixture.rowSetBuilder(schema)
+ .addSingleCol(strArray("sixth", "6.1", "6.2"))
+ .addSingleCol(strArray("second", "2.1", "2.2", "2.3"))
+ .addSingleCol(strArray("fourth", "4.1"))
+ .build();
+
+ SingleRowSet rowSet2 = fixture.rowSetBuilder(schema)
+ .addSingleCol(strArray("fifth", "51", "5.2"))
+ .addSingleCol(strArray("first", "1.1", "1.2", "1.3"))
+ .addSingleCol(strArray("third", "3.1"))
+ .build();
+
+ // Build the hyper batch
+
+ HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2);
+ assertEquals(6, hyperSet.rowCount());
+ SelectionVector4 sv4 = hyperSet.getSv4();
+ sv4.set(0, 1, 1);
+ sv4.set(1, 0, 1);
+ sv4.set(2, 1, 2);
+ sv4.set(3, 0, 2);
+ sv4.set(4, 1, 0);
+ sv4.set(5, 0, 0);
+
+ SingleRowSet expected = fixture.rowSetBuilder(schema)
+ .addSingleCol(strArray("first", "1.1", "1.2", "1.3"))
+ .addSingleCol(strArray("second", "2.1", "2.2", "2.3"))
+ .addSingleCol(strArray("third", "3.1"))
+ .addSingleCol(strArray("fourth", "4.1"))
+ .addSingleCol(strArray("fifth", "51", "5.2"))
+ .addSingleCol(strArray("sixth", "6.1", "6.2"))
+ .build();
+
+ RowSetUtilities.verify(expected, hyperSet);
+ }
+
+ /**
+ * Maps are an interesting case. The hyper-vector wrapper holds a mirror-image of the
+ * map members. So, we can reach the map members either via the vector wrappers or
+ * the original map vector.
+ */
+
+ @Test
+ public void testMap() {
+ TupleMetadata schema = new SchemaBuilder()
+ .addMap("m")
+ .add("a", MinorType.INT)
+ .add("b", MinorType.VARCHAR)
+ .resumeSchema()
+ .buildSchema();
+
+ SingleRowSet rowSet1 = fixture.rowSetBuilder(schema)
+ .addSingleCol(mapValue(2, "second"))
+ .addSingleCol(mapValue(4, "fourth"))
+ .build();
+
+ SingleRowSet rowSet2 = fixture.rowSetBuilder(schema)
+ .addSingleCol(mapValue(2, "first"))
+ .addSingleCol(mapValue(4, "third"))
+ .build();
+
+ // Build the hyper batch
+
+ HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2);
+ assertEquals(4, hyperSet.rowCount());
+ SelectionVector4 sv4 = hyperSet.getSv4();
+ sv4.set(0, 1, 0);
+ sv4.set(1, 0, 0);
+ sv4.set(2, 1, 1);
+ sv4.set(3, 0, 1);
+
+ SingleRowSet expected = fixture.rowSetBuilder(schema)
+ .addSingleCol(mapValue(2, "first"))
+ .addSingleCol(mapValue(2, "second"))
+ .addSingleCol(mapValue(4, "third"))
+ .addSingleCol(mapValue(4, "fourth"))
+ .build();
+
+ RowSetUtilities.verify(expected, hyperSet);
+ }
+
+ @Test
+ public void testRepeatedMap() {
+ TupleMetadata schema = new SchemaBuilder()
+ .add("a", MinorType.INT)
+ .addMapArray("ma")
+ .add("b", MinorType.INT)
+ .add("c", MinorType.VARCHAR)
+ .resumeSchema()
+ .buildSchema();
+
+ SingleRowSet rowSet1 = fixture.rowSetBuilder(schema)
+ .addRow(2, mapArray(mapValue(21, "second.1"), mapValue(22, "second.2")))
+ .addRow(4, mapArray(mapValue(41, "fourth.1")))
+ .build();
+
+ SingleRowSet rowSet2 = fixture.rowSetBuilder(schema)
+ .addRow(1, mapArray(mapValue(11, "first.1"), mapValue(12, "first.2")))
+ .addRow(3, mapArray(mapValue(31, "third.1"), mapValue(32, "third.2"), mapValue(33, "third.3")))
+ .build();
+
+ // Build the hyper batch
+
+ HyperRowSet hyperSet = HyperRowSetImpl.fromRowSets(fixture.allocator(), rowSet1, rowSet2);
+ assertEquals(4, hyperSet.rowCount());
+ SelectionVector4 sv4 = hyperSet.getSv4();
+ sv4.set(0, 1, 0);
+ sv4.set(1, 0, 0);
+ sv4.set(2, 1, 1);
+ sv4.set(3, 0, 1);
+
+ SingleRowSet expected = fixture.rowSetBuilder(schema)
+ .addRow(1, mapArray(mapValue(11, "first.1"), mapValue(12, "first.2")))
+ .addRow(2, mapArray(mapValue(21, "second.1"), mapValue(22, "second.2")))
+ .addRow(3, mapArray(mapValue(31, "third.1"), mapValue(32, "third.2"), mapValue(33, "third.3")))
+ .addRow(4, mapArray(mapValue(41, "fourth.1")))
+ .build();
+
+ RowSetUtilities.verify(expected, hyperSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java
index 1daeef4..ecd1ebc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java
@@ -26,7 +26,8 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
+import org.apache.drill.exec.vector.DateUtilities;
+import org.apache.drill.exec.vector.accessor.ArrayReader;
import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.test.SubOperatorTest;
@@ -36,6 +37,8 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
import org.apache.drill.test.rowSet.schema.SchemaBuilder;
import org.junit.Test;
+import com.google.common.collect.Lists;
+
/**
* Verify that simple scalar (non-repeated) column readers
* and writers work as expected. The focus is on the generated
@@ -51,6 +54,70 @@ import org.junit.Test;
public class TestScalarAccessors extends SubOperatorTest {
@Test
+ public void testUInt1RW() {
+ BatchSchema batchSchema = new SchemaBuilder()
+ .add("col", MinorType.UINT1)
+ .build();
+ SingleRowSet rs = fixture.rowSetBuilder(batchSchema)
+ .addRow(0)
+ .addRow(0x7F)
+ .addRow(0xFF)
+ .build();
+ assertEquals(3, rs.rowCount());
+
+ RowSetReader reader = rs.reader();
+ ScalarReader colReader = reader.scalar(0);
+ assertEquals(ValueType.INTEGER, colReader.valueType());
+
+ assertTrue(reader.next());
+ assertFalse(colReader.isNull());
+ assertEquals(0, colReader.getInt());
+
+ assertTrue(reader.next());
+ assertEquals(0x7F, colReader.getInt());
+ assertEquals(0x7F, colReader.getObject());
+ assertEquals(Integer.toString(0x7F), colReader.getAsString());
+
+ assertTrue(reader.next());
+ assertEquals(0xFF, colReader.getInt());
+
+ assertFalse(reader.next());
+ rs.clear();
+ }
+
+ @Test
+ public void testUInt2RW() {
+ BatchSchema batchSchema = new SchemaBuilder()
+ .add("col", MinorType.UINT2)
+ .build();
+ SingleRowSet rs = fixture.rowSetBuilder(batchSchema)
+ .addRow(0)
+ .addRow(0x7FFF)
+ .addRow(0xFFFF)
+ .build();
+ assertEquals(3, rs.rowCount());
+
+ RowSetReader reader = rs.reader();
+ ScalarReader colReader = reader.scalar(0);
+ assertEquals(ValueType.INTEGER, colReader.valueType());
+
+ assertTrue(reader.next());
+ assertFalse(colReader.isNull());
+ assertEquals(0, colReader.getInt());
+
+ assertTrue(reader.next());
+ assertEquals(0x7FFF, colReader.getInt());
+ assertEquals(0x7FFF, colReader.getObject());
+ assertEquals(Integer.toString(0x7FFF), colReader.getAsString());
+
+ assertTrue(reader.next());
+ assertEquals(0xFFFF, colReader.getInt());
+
+ assertFalse(reader.next());
+ rs.clear();
+ }
+
+ @Test
public void testTinyIntRW() {
BatchSchema batchSchema = new SchemaBuilder()
.add("col", MinorType.TINYINT)
@@ -129,23 +196,37 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.INTEGER, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(0, colReader.getInt(0));
- assertEquals(20, colReader.getInt(1));
- assertEquals(30, colReader.getInt(2));
- assertEquals(0, colReader.getObject(0));
- assertEquals(20, colReader.getObject(1));
- assertEquals(30, colReader.getObject(2));
- assertEquals("0", colReader.getAsString(0));
- assertEquals("20", colReader.getAsString(1));
- assertEquals("30", colReader.getAsString(2));
+ assertEquals(3, arrayReader.size());
+ assertTrue(arrayReader.next());
+ assertFalse(colReader.isNull());
+ assertEquals(0, colReader.getInt());
+ assertEquals(0, colReader.getObject());
+ assertEquals("0", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertFalse(colReader.isNull());
+ assertEquals(20, colReader.getInt());
+ assertEquals(20, colReader.getObject());
+ assertEquals("20", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertFalse(colReader.isNull());
+ assertEquals(30, colReader.getInt());
+ assertEquals(30, colReader.getObject());
+ assertEquals("30", colReader.getAsString());
+
+ assertFalse(arrayReader.next());
+
+ assertEquals("[0, 20, 30]", arrayReader.getAsString());
+ assertEquals(Lists.newArrayList(0, 20, 30), arrayReader.getObject());
assertFalse(reader.next());
rs.clear();
@@ -323,23 +404,35 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.LONG, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(0, colReader.getLong(0));
- assertEquals(20, colReader.getLong(1));
- assertEquals(30, colReader.getLong(2));
- assertEquals(0L, colReader.getObject(0));
- assertEquals(20L, colReader.getObject(1));
- assertEquals(30L, colReader.getObject(2));
- assertEquals("0", colReader.getAsString(0));
- assertEquals("20", colReader.getAsString(1));
- assertEquals("30", colReader.getAsString(2));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(0, colReader.getLong());
+ assertEquals(0L, colReader.getObject());
+ assertEquals("0", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(20, colReader.getLong());
+ assertEquals(20L, colReader.getObject());
+ assertEquals("20", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(30, colReader.getLong());
+ assertEquals(30L, colReader.getObject());
+ assertEquals("30", colReader.getAsString());
+
+ assertFalse(arrayReader.next());
+
+ assertEquals("[0, 20, 30]", arrayReader.getAsString());
+ assertEquals(Lists.newArrayList(0L, 20L, 30L), arrayReader.getObject());
assertFalse(reader.next());
rs.clear();
@@ -373,7 +466,7 @@ public class TestScalarAccessors extends SubOperatorTest {
assertTrue(reader.next());
assertEquals(Float.MAX_VALUE, colReader.getDouble(), 0.000001);
- assertEquals((double) Float.MAX_VALUE, (double) colReader.getObject(), 0.000001);
+ assertEquals(Float.MAX_VALUE, (double) colReader.getObject(), 0.000001);
assertTrue(reader.next());
assertEquals(Float.MIN_VALUE, colReader.getDouble(), 0.000001);
@@ -433,23 +526,34 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.DOUBLE, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(0, colReader.getDouble(0), 0.00001);
- assertEquals(20.5, colReader.getDouble(1), 0.00001);
- assertEquals(30.0, colReader.getDouble(2), 0.00001);
- assertEquals(0, (double) colReader.getObject(0), 0.00001);
- assertEquals(20.5, (double) colReader.getObject(1), 0.00001);
- assertEquals(30.0, (double) colReader.getObject(2), 0.00001);
- assertEquals("0.0", colReader.getAsString(0));
- assertEquals("20.5", colReader.getAsString(1));
- assertEquals("30.0", colReader.getAsString(2));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(0, colReader.getDouble(), 0.00001);
+ assertEquals(0, (double) colReader.getObject(), 0.00001);
+ assertEquals("0.0", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(20.5, colReader.getDouble(), 0.00001);
+ assertEquals(20.5, (double) colReader.getObject(), 0.00001);
+ assertEquals("20.5", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(30.0, colReader.getDouble(), 0.00001);
+ assertEquals(30.0, (double) colReader.getObject(), 0.00001);
+ assertEquals("30.0", colReader.getAsString());
+ assertFalse(arrayReader.next());
+
+ assertEquals("[0.0, 20.5, 30.0]", arrayReader.getAsString());
+ assertEquals(Lists.newArrayList(0.0D, 20.5D, 30D), arrayReader.getObject());
assertFalse(reader.next());
rs.clear();
@@ -507,15 +611,16 @@ public class TestScalarAccessors extends SubOperatorTest {
}
@Test
- public void testStringRW() {
+ public void testVarcharRW() {
BatchSchema batchSchema = new SchemaBuilder()
.add("col", MinorType.VARCHAR)
.build();
SingleRowSet rs = fixture.rowSetBuilder(batchSchema)
.addRow("")
- .addRow("abcd")
+ .addRow("fred")
+ .addRow("barney")
.build();
- assertEquals(2, rs.rowCount());
+ assertEquals(3, rs.rowCount());
RowSetReader reader = rs.reader();
ScalarReader colReader = reader.scalar(0);
@@ -526,16 +631,21 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals("", colReader.getString());
assertTrue(reader.next());
- assertEquals("abcd", colReader.getString());
- assertEquals("abcd", colReader.getObject());
- assertEquals("\"abcd\"", colReader.getAsString());
+ assertEquals("fred", colReader.getString());
+ assertEquals("fred", colReader.getObject());
+ assertEquals("\"fred\"", colReader.getAsString());
+
+ assertTrue(reader.next());
+ assertEquals("barney", colReader.getString());
+ assertEquals("barney", colReader.getObject());
+ assertEquals("\"barney\"", colReader.getAsString());
assertFalse(reader.next());
rs.clear();
}
@Test
- public void testNullableString() {
+ public void testNullableVarchar() {
BatchSchema batchSchema = new SchemaBuilder()
.addNullable("col", MinorType.VARCHAR)
.build();
@@ -568,7 +678,7 @@ public class TestScalarAccessors extends SubOperatorTest {
}
@Test
- public void testStringArray() {
+ public void testVarcharArray() {
BatchSchema batchSchema = new SchemaBuilder()
.addArray("col", MinorType.VARCHAR)
.build();
@@ -579,28 +689,71 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.STRING, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals("fred", colReader.getString(0));
- assertEquals("", colReader.getString(1));
- assertEquals("wilma", colReader.getString(2));
- assertEquals("fred", colReader.getObject(0));
- assertEquals("", colReader.getObject(1));
- assertEquals("wilma", colReader.getObject(2));
- assertEquals("\"fred\"", colReader.getAsString(0));
- assertEquals("\"\"", colReader.getAsString(1));
- assertEquals("\"wilma\"", colReader.getAsString(2));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals("fred", colReader.getString());
+ assertEquals("fred", colReader.getObject());
+ assertEquals("\"fred\"", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals("", colReader.getString());
+ assertEquals("", colReader.getObject());
+ assertEquals("\"\"", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals("wilma", colReader.getString());
+ assertEquals("wilma", colReader.getObject());
+ assertEquals("\"wilma\"", colReader.getAsString());
+
+ assertFalse(arrayReader.next());
+
+ assertEquals("[\"fred\", \"\", \"wilma\"]", arrayReader.getAsString());
+ assertEquals(Lists.newArrayList("fred", "", "wilma"), arrayReader.getObject());
assertFalse(reader.next());
rs.clear();
}
+ /**
+ * Test the low-level interval-year utilities used by the column accessors.
+ */
+
+ @Test
+ public void testIntervalYearUtils() {
+ {
+ Period expected = Period.months(0);
+ Period actual = DateUtilities.fromIntervalYear(0);
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalYearStringBuilder(expected).toString();
+ assertEquals("0 years 0 months", fmt);
+ }
+
+ {
+ Period expected = Period.years(1).plusMonths(2);
+ Period actual = DateUtilities.fromIntervalYear(DateUtilities.periodToMonths(expected));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalYearStringBuilder(expected).toString();
+ assertEquals("1 year 2 months", fmt);
+ }
+
+ {
+ Period expected = Period.years(6).plusMonths(1);
+ Period actual = DateUtilities.fromIntervalYear(DateUtilities.periodToMonths(expected));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalYearStringBuilder(expected).toString();
+ assertEquals("6 years 1 month", fmt);
+ }
+ }
+
@Test
public void testIntervalYearRW() {
BatchSchema batchSchema = new SchemaBuilder()
@@ -664,7 +817,6 @@ public class TestScalarAccessors extends SubOperatorTest {
assertTrue(reader.next());
assertTrue(colReader.isNull());
- assertNull(colReader.getPeriod());
assertNull(colReader.getObject());
assertEquals("null", colReader.getAsString());
@@ -692,24 +844,64 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.PERIOD, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(p1, colReader.getPeriod(0));
- assertEquals(p2, colReader.getPeriod(1));
- assertEquals(p3, colReader.getPeriod(2));
- assertEquals(p2, colReader.getObject(1));
- assertEquals(p2.toString(), colReader.getAsString(1));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p1, colReader.getPeriod());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p2, colReader.getPeriod());
+ assertEquals(p2, colReader.getObject());
+ assertEquals(p2.toString(), colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p3, colReader.getPeriod());
+
+ assertFalse(arrayReader.next());
assertFalse(reader.next());
rs.clear();
}
+ /**
+ * Test the low-level interval-day utilities used by the column accessors.
+ */
+
+ @Test
+ public void testIntervalDayUtils() {
+ {
+ Period expected = Period.days(0);
+ Period actual = DateUtilities.fromIntervalDay(0, 0);
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalDayStringBuilder(expected).toString();
+ assertEquals("0 days 0:00:00", fmt);
+ }
+
+ {
+ Period expected = Period.days(1).plusHours(5).plusMinutes(6).plusSeconds(7);
+ Period actual = DateUtilities.fromIntervalDay(1, DateUtilities.timeToMillis(5, 6, 7, 0));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalDayStringBuilder(expected).toString();
+ assertEquals("1 day 5:06:07", fmt);
+ }
+
+ {
+ Period expected = Period.days(2).plusHours(12).plusMinutes(23).plusSeconds(34).plusMillis(567);
+ Period actual = DateUtilities.fromIntervalDay(2, DateUtilities.timeToMillis(12, 23, 34, 567));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalDayStringBuilder(expected).toString();
+ assertEquals("2 days 12:23:34.567", fmt);
+ }
+ }
+
@Test
public void testIntervalDayRW() {
BatchSchema batchSchema = new SchemaBuilder()
@@ -774,7 +966,6 @@ public class TestScalarAccessors extends SubOperatorTest {
assertTrue(reader.next());
assertTrue(colReader.isNull());
- assertNull(colReader.getPeriod());
assertNull(colReader.getObject());
assertEquals("null", colReader.getAsString());
@@ -802,24 +993,79 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.PERIOD, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(p1, colReader.getPeriod(0).normalizedStandard());
- assertEquals(p2, colReader.getPeriod(1).normalizedStandard());
- assertEquals(p3.normalizedStandard(), colReader.getPeriod(2).normalizedStandard());
- assertEquals(p2, ((Period) colReader.getObject(1)).normalizedStandard());
- assertEquals(p2.toString(), colReader.getAsString(1));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p1, colReader.getPeriod().normalizedStandard());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p2, colReader.getPeriod().normalizedStandard());
+ assertEquals(p2, ((Period) colReader.getObject()).normalizedStandard());
+ assertEquals(p2.toString(), colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p3.normalizedStandard(), colReader.getPeriod().normalizedStandard());
+
+ assertFalse(arrayReader.next());
assertFalse(reader.next());
rs.clear();
}
+ /**
+ * Test the low-level interval utilities used by the column accessors.
+ */
+
+ @Test
+ public void testIntervalUtils() {
+ {
+ Period expected = Period.months(0);
+ Period actual = DateUtilities.fromInterval(0, 0, 0);
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalStringBuilder(expected).toString();
+ assertEquals("0 years 0 months 0 days 0:00:00", fmt);
+ }
+
+ {
+ Period expected = Period.years(1).plusMonths(2).plusDays(3)
+ .plusHours(5).plusMinutes(6).plusSeconds(7);
+ Period actual = DateUtilities.fromInterval(DateUtilities.periodToMonths(expected), 3,
+ DateUtilities.periodToMillis(expected));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalStringBuilder(expected).toString();
+ assertEquals("1 year 2 months 3 days 5:06:07", fmt);
+ }
+
+ {
+ Period expected = Period.years(2).plusMonths(1).plusDays(3)
+ .plusHours(12).plusMinutes(23).plusSeconds(34)
+ .plusMillis(456);
+ Period actual = DateUtilities.fromInterval(DateUtilities.periodToMonths(expected), 3,
+ DateUtilities.periodToMillis(expected));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalStringBuilder(expected).toString();
+ assertEquals("2 years 1 month 3 days 12:23:34.456", fmt);
+ }
+
+ {
+ Period expected = Period.years(2).plusMonths(3).plusDays(1)
+ .plusHours(12).plusMinutes(23).plusSeconds(34);
+ Period actual = DateUtilities.fromInterval(DateUtilities.periodToMonths(expected), 1,
+ DateUtilities.periodToMillis(expected));
+ assertEquals(expected, actual.normalizedStandard());
+ String fmt = DateUtilities.intervalStringBuilder(expected).toString();
+ assertEquals("2 years 3 months 1 day 12:23:34", fmt);
+ }
+ }
+
@Test
public void testIntervalRW() {
BatchSchema batchSchema = new SchemaBuilder()
@@ -890,7 +1136,6 @@ public class TestScalarAccessors extends SubOperatorTest {
assertTrue(reader.next());
assertTrue(colReader.isNull());
- assertNull(colReader.getPeriod());
assertNull(colReader.getObject());
assertEquals("null", colReader.getAsString());
@@ -922,19 +1167,28 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.PERIOD, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(p1, colReader.getPeriod(0).normalizedStandard());
- assertEquals(p2, colReader.getPeriod(1).normalizedStandard());
- assertEquals(p3.normalizedStandard(), colReader.getPeriod(2).normalizedStandard());
- assertEquals(p2, ((Period) colReader.getObject(1)).normalizedStandard());
- assertEquals(p2.toString(), colReader.getAsString(1));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p1, colReader.getPeriod().normalizedStandard());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p2, colReader.getPeriod().normalizedStandard());
+ assertEquals(p2, ((Period) colReader.getObject()).normalizedStandard());
+ assertEquals(p2.toString(), colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(p3.normalizedStandard(), colReader.getPeriod().normalizedStandard());
+
+ assertFalse(arrayReader.next());
assertFalse(reader.next());
rs.clear();
@@ -1051,19 +1305,28 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.DECIMAL, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertEquals(0, v1.compareTo(colReader.getDecimal(0)));
- assertEquals(0, v2.compareTo(colReader.getDecimal(1)));
- assertEquals(0, v3.compareTo(colReader.getDecimal(2)));
- assertEquals(0, v2.compareTo((BigDecimal) colReader.getObject(1)));
- assertEquals(v2.toString(), colReader.getAsString(1));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertEquals(0, v1.compareTo(colReader.getDecimal()));
+
+ assertTrue(arrayReader.next());
+ assertEquals(0, v2.compareTo(colReader.getDecimal()));
+ assertEquals(0, v2.compareTo((BigDecimal) colReader.getObject()));
+ assertEquals(v2.toString(), colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertEquals(0, v3.compareTo(colReader.getDecimal()));
+
+ assertFalse(arrayReader.next());
assertFalse(reader.next());
rs.clear();
@@ -1246,19 +1509,28 @@ public class TestScalarAccessors extends SubOperatorTest {
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
- ScalarElementReader colReader = reader.elements(0);
+ ArrayReader arrayReader = reader.array(0);
+ ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.BYTES, colReader.valueType());
assertTrue(reader.next());
- assertEquals(0, colReader.size());
+ assertEquals(0, arrayReader.size());
assertTrue(reader.next());
- assertEquals(3, colReader.size());
- assertTrue(Arrays.equals(v1, colReader.getBytes(0)));
- assertTrue(Arrays.equals(v2, colReader.getBytes(1)));
- assertTrue(Arrays.equals(v3, colReader.getBytes(2)));
- assertTrue(Arrays.equals(v2, (byte[]) colReader.getObject(1)));
- assertEquals("[00, 7f, 80, ff]", colReader.getAsString(1));
+ assertEquals(3, arrayReader.size());
+
+ assertTrue(arrayReader.next());
+ assertTrue(Arrays.equals(v1, colReader.getBytes()));
+
+ assertTrue(arrayReader.next());
+ assertTrue(Arrays.equals(v2, colReader.getBytes()));
+ assertTrue(Arrays.equals(v2, (byte[]) colReader.getObject()));
+ assertEquals("[00, 7f, 80, ff]", colReader.getAsString());
+
+ assertTrue(arrayReader.next());
+ assertTrue(Arrays.equals(v3, colReader.getBytes()));
+
+ assertFalse(arrayReader.next());
assertFalse(reader.next());
rs.clear();