You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/06/11 15:13:58 UTC
[10/12] arrow git commit: ARROW-1111: [JAVA] Make aligning buffers
optional, and allow -1 for unknown null count
ARROW-1111: [JAVA] Make aligning buffers optional, and allow -1 for unknown null count
Closes #747
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/11deee61
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/11deee61
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/11deee61
Branch: refs/heads/master
Commit: 11deee61aa2fd6782df1f8eefa15938a7676a546
Parents: c6cf124
Author: Steven Phillips <st...@dremio.com>
Authored: Tue Jun 6 15:19:00 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/arrow/vector/VectorUnloader.java | 12 ++++++++++--
.../apache/arrow/vector/schema/ArrowRecordBatch.java | 8 ++++++--
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/11deee61/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 8e9ff6d..9965245 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -29,9 +29,17 @@ import org.apache.arrow.vector.schema.ArrowVectorType;
public class VectorUnloader {
private final VectorSchemaRoot root;
+ private final boolean includeNullCount;
+ private final boolean alignBuffers;
public VectorUnloader(VectorSchemaRoot root) {
+ this(root, true, true);
+ }
+
+ public VectorUnloader(VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers) {
this.root = root;
+ this.includeNullCount = includeNullCount;
+ this.alignBuffers = alignBuffers;
}
public ArrowRecordBatch getRecordBatch() {
@@ -40,12 +48,12 @@ public class VectorUnloader {
for (FieldVector vector : root.getFieldVectors()) {
appendNodes(vector, nodes, buffers);
}
- return new ArrowRecordBatch(root.getRowCount(), nodes, buffers);
+ return new ArrowRecordBatch(root.getRowCount(), nodes, buffers, alignBuffers);
}
private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
Accessor accessor = vector.getAccessor();
- nodes.add(new ArrowFieldNode(accessor.getValueCount(), accessor.getNullCount()));
+ nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1));
List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
if (fieldBuffers.size() != expectedBuffers.size()) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/11deee61/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 6ef514e..6a716fa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -48,12 +48,16 @@ public class ArrowRecordBatch implements ArrowMessage {
private boolean closed = false;
+ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+ this(length, nodes, buffers, true);
+ }
+
/**
* @param length how many rows in this batch
* @param nodes field level info
* @param buffers will be retained until this recordBatch is closed
*/
- public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers, boolean alignBuffers) {
super();
this.length = length;
this.nodes = nodes;
@@ -66,7 +70,7 @@ public class ArrowRecordBatch implements ArrowMessage {
arrowBuffers.add(new ArrowBuffer(0, offset, size));
LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size));
offset += size;
- if (offset % 8 != 0) { // align on 8 byte boundaries
+ if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries
offset += 8 - (offset % 8);
}
}