You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by cwestin <gi...@git.apache.org> on 2015/10/09 19:47:46 UTC
[GitHub] drill pull request: DRILL-1942-vector-test
GitHub user cwestin opened a pull request:
https://github.com/apache/drill/pull/194
DRILL-1942-vector-test
Additional tests added to TestValueVectors for serialization and loading; these tests were used to find and debug issues with DrillBuf slicing. Some light cleanup of a few vector implementations.
Unit tests pass.
Regression suite has a few (known recurring) failures:
- timeouts while waiting to send intermediate work fragments
- timeouts for "Fetch parquet metadata" tasks not complete
- Selected column 'dir0' must have name 'columns' or must be plain '*'
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/cwestin/incubator-drill alloc-vector-test
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/drill/pull/194.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #194
----
commit 3b5e0bd007031766d4e3eb486a51c5a8cbcbc026
Author: Chris Westin <cw...@yahoo.com>
Date: 2015-10-09T01:15:57Z
DRILL-1942-vector-test
Additional tests added to TestValueVectors for serialization and loading.
Some light cleanup of a few vector implementations.
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41809607
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ vector1.close();
+ vector2.close();
+ buffer1.release();
+ }
+
+ @Test
+ public void testNullableVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 nullable variable length strings.
+ final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+ final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+
+ // Check the contents.
+ final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Still ok after setting value count?
+ stringBuilder.setLength(0);
--- End diff --
do we really need this check on vector1 as we already check this on vector2? we should consider removing this.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41808581
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -174,9 +179,9 @@ public void testVariableVectorReallocation() {
public void testFixedType() {
MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
--- End diff --
final
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-1942-vector-test
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on the pull request:
https://github.com/apache/drill/pull/194#issuecomment-147255263
This is now https://issues.apache.org/jira/browse/DRILL-3920 .
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41808424
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---
@@ -355,17 +353,19 @@ public void generateTestData(int values) { }
@Override
public void clear() {
- valueCount = 0;
- for (ValueVector v : getChildren()) {
+ for (final ValueVector v : getChildren()) {
--- End diff --
the logic in clear() seems duplicating close(). we should directly call close perhaps.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41814111
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ vector1.close();
+ vector2.close();
+ buffer1.release();
+ }
+
+ @Test
+ public void testNullableVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 nullable variable length strings.
+ final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+ final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+
+ // Check the contents.
+ final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Still ok after setting value count?
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ System.out.println(object);
--- End diff --
if we are keeping this block, we don't need this print statement as well.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41809364
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
--- End diff --
we should remove this.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41907849
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---
@@ -355,17 +353,19 @@ public void generateTestData(int values) { }
@Override
public void clear() {
- valueCount = 0;
- for (ValueVector v : getChildren()) {
+ for (final ValueVector v : getChildren()) {
v.clear();
}
+ valueCount = 0;
}
@Override
public void close() {
- for (final ValueVector v : getChildren()) {
+ final Collection<ValueVector> vectors = getChildren();
--- End diff --
This change seems risky, given that everything passed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41812282
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -174,9 +179,9 @@ public void testVariableVectorReallocation() {
public void testFixedType() {
MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
--- End diff --
Done.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on the pull request:
https://github.com/apache/drill/pull/194#issuecomment-147879238
+1
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/drill/pull/194
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41908196
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
--- End diff --
println removed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-1942-vector-test
Posted by jacques-n <gi...@git.apache.org>.
Github user jacques-n commented on the pull request:
https://github.com/apache/drill/pull/194#issuecomment-147252419
Can you break each of these commits into its own jira? It is confusing that you are doing a bunch of independent commits under the same jira number.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41908273
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ vector1.close();
+ vector2.close();
+ buffer1.release();
+ }
+
+ @Test
+ public void testNullableVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 nullable variable length strings.
+ final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+ final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+
+ // Check the contents.
+ final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Still ok after setting value count?
+ stringBuilder.setLength(0);
--- End diff --
It doesn't need to be in both places.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41812350
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
--- End diff --
I don't see how that handles multiple buffers.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41808483
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---
@@ -355,17 +353,19 @@ public void generateTestData(int values) { }
@Override
public void clear() {
- valueCount = 0;
- for (ValueVector v : getChildren()) {
+ for (final ValueVector v : getChildren()) {
v.clear();
}
+ valueCount = 0;
}
@Override
public void close() {
- for (final ValueVector v : getChildren()) {
+ final Collection<ValueVector> vectors = getChildren();
--- End diff --
super.close() and resetting value count should be just fine here. remaining seems redundant if I am not missing anything.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41808984
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
--- End diff --
not a show stopper but the whole loop here could be replaced with ByteBuf#readBytes or ByteBuf#getBytes I think.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41814004
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ vector1.close();
+ vector2.close();
+ buffer1.release();
+ }
+
+ @Test
+ public void testNullableVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 nullable variable length strings.
+ final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+ final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+
+ // Check the contents.
+ final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Still ok after setting value count?
+ stringBuilder.setLength(0);
--- End diff --
If you think this is really useful. We should consider adding the same check to the test above testVarCharVectorLoad then.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41908307
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ vector1.close();
+ vector2.close();
+ buffer1.release();
+ }
+
+ @Test
+ public void testNullableVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 nullable variable length strings.
+ final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+ final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+
+ // Check the contents.
+ final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Still ok after setting value count?
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ System.out.println(object);
--- End diff --
println() removed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41908122
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
--- End diff --
Done.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41907790
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---
@@ -355,17 +353,19 @@ public void generateTestData(int values) { }
@Override
public void clear() {
- valueCount = 0;
- for (ValueVector v : getChildren()) {
+ for (final ValueVector v : getChildren()) {
--- End diff --
But clear() doesn't call vectors.clear(). I seem to recall a problem with adding that to clear() in the past. I'm worried this will introduce a regression, so I'd rather not take that change on now.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41811855
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
--- End diff --
Why? This tests something that wasn't working at one point.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41813759
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
--- End diff --
System.out.println(object)?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41808557
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -174,9 +179,9 @@ public void testVariableVectorReallocation() {
public void testFixedType() {
MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
--- End diff --
final
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on the pull request:
https://github.com/apache/drill/pull/194#issuecomment-147814132
I made the changes indicated above, but not any that might destabilize tests (noted above). Please re-review and merge.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by hnfgns <gi...@git.apache.org>.
Github user hnfgns commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41813915
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
--- End diff --
It does not. I meant that you could replace the inner loop where you write individual bytes into writeBuf. You could make the same using read/getBytes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] drill pull request: DRILL-3920 Add vector loading tests
Posted by cwestin <gi...@git.apache.org>.
Github user cwestin commented on a diff in the pull request:
https://github.com/apache/drill/pull/194#discussion_r41811916
--- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---
@@ -223,14 +231,187 @@ public void testNullableVarLen2() {
}
}
+ private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[] buffers) {
+ // find the total size we'll need
+ int size = 0;
+ for(final DrillBuf buffer : buffers) {
+ size += buffer.readableBytes();
+ }
+
+ // create the new buffer
+ final DrillBuf newBuf = allocator.buffer(size);
+ final DrillBuf writeBuf = newBuf;
+ for(final DrillBuf buffer : buffers) {
+ final DrillBuf readBuf = (DrillBuf) buffer.slice();
+ final int nBytes = readBuf.readableBytes();
+ for(int i = 0; i < nBytes; ++i) {
+ writeBuf.writeByte(readBuf.readByte());
+ }
+ }
+
+ return newBuf;
+ }
+
+ @Test
+ public void testRepeatedIntVector() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+ // Create a new value vector.
+ final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+ // Populate the vector.
+ final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+ final int nRecords = 7;
+ final int nElements = values.length;
+ vector1.allocateNew(nRecords, nRecords * nElements);
+ final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ mutator.startNewValue(recordIndex);
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ mutator.add(recordIndex, recordIndex * values[elementIndex]);
+ }
+ }
+ mutator.setValueCount(nRecords);
+
+ // Verify the contents.
+ final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+ assertEquals(nRecords, accessor1.getValueCount());
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor1.get(recordIndex, elementIndex);
+ assertEquals(recordIndex * values[elementIndex], value);
+ }
+ }
+
+/* TODO(cwestin)
+the interface to load has changed
+ // Serialize, reify, and verify.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+ vector2.load(nRecords, nRecords * nElements, buffer1);
+
+ final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+ for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+ for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+ final int value = accessor2.get(recordIndex, elementIndex);
+ assertEquals(accessor1.get(recordIndex, elementIndex), value);
+ }
+ }
+*/
+
+ vector1.close();
+/* TODO(cwestin)
+ vector2.close();
+ buffer1.release();
+*/
+ }
+
+ @Test
+ public void testVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 variable length strings.
+ final VarCharVector vector1 = new VarCharVector(field, allocator);
+ final VarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Combine the backing buffers so we can load them into a new vector.
+ final DrillBuf[] buffers1 = vector1.getBuffers(false);
+ final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+ final VarCharVector vector2 = new VarCharVector(field, allocator);
+ vector2.load(vector1.getMetadata(), buffer1);
+
+ // Check the contents of the new vector.
+ final VarCharVector.Accessor accessor = vector2.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor.getObject(i);
+ System.out.println(object);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ vector1.close();
+ vector2.close();
+ buffer1.release();
+ }
+
+ @Test
+ public void testNullableVarCharVectorLoad() {
+ final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+ // Create a new value vector for 1024 nullable variable length strings.
+ final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+ final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+ vector1.allocateNew(1024 * 10, 1024);
+
+ // Populate the vector.
+ final StringBuilder stringBuilder = new StringBuilder();
+ final int valueCount = 10;
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+ }
+
+ // Check the contents.
+ final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+ stringBuilder.setLength(0);
+ for(int i = 0; i < valueCount; ++i) {
+ stringBuilder.append('x');
+ final Object object = accessor1.getObject(i);
+ assertEquals(stringBuilder.toString(), object.toString());
+ }
+
+ mutator.setValueCount(valueCount);
+ assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+ // Still ok after setting value count?
+ stringBuilder.setLength(0);
--- End diff --
This tests to make sure we can read back what we wrote. Why remove that?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---