You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/06/11 15:13:49 UTC
[01/12] arrow git commit: [maven-release-plugin] prepare release
apache-arrow-0.4.1 [Forced Update!]
Repository: arrow
Updated Branches:
refs/heads/master 037d78dda -> ac6485390 (forced update)
[maven-release-plugin] prepare release apache-arrow-0.4.1
Change-Id: I36c4db377ed8cfc10de14604114a79454db10aa1
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/46315431
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/46315431
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/46315431
Branch: refs/heads/master
Commit: 46315431aeda3b6968b3ac4c1087f6d41052b99d
Parents: 2a12482
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue Jun 6 22:13:26 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue Jun 6 22:13:26 2017 -0400
----------------------------------------------------------------------
java/format/pom.xml | 2 +-
java/memory/pom.xml | 2 +-
java/pom.xml | 4 ++--
java/tools/pom.xml | 2 +-
java/vector/pom.xml | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/46315431/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
index e092754..9d509bc 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>arrow-java-root</artifactId>
<groupId>org.apache.arrow</groupId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.4.1</version>
</parent>
<artifactId>arrow-format</artifactId>
http://git-wip-us.apache.org/repos/asf/arrow/blob/46315431/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index dc4d0da..6a8fab0 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.4.1</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>Arrow Memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/46315431/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 1ec3d56..094914b 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.4.1</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
<url>https://github.com/apache/arrow</url>
- <tag>HEAD</tag>
+ <tag>apache-arrow-0.4.1</tag>
</scm>
<mailingLists>
http://git-wip-us.apache.org/repos/asf/arrow/blob/46315431/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 53331f2..f93cef8 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.4.1</version>
</parent>
<artifactId>arrow-tools</artifactId>
<name>Arrow Tools</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/46315431/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index a117a2f..60e342d 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.5.0-SNAPSHOT</version>
+ <version>0.4.1</version>
</parent>
<artifactId>arrow-vector</artifactId>
<name>Arrow Vectors</name>
[03/12] arrow git commit: ARROW-1102: Make
MessageSerializer.serializeMessage() public
Posted by we...@apache.org.
ARROW-1102: Make MessageSerializer.serializeMessage() public
Author: Julien Le Dem <ju...@apache.org>
Closes #741 from julienledem/ARROW-1102 and squashes the following commits:
284eaaf1 [Julien Le Dem] ARROW-1102: Make MessageSerializer.serializeMessage() public
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a44d5842
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a44d5842
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a44d5842
Branch: refs/heads/master
Commit: a44d5842ce372d787050dc7aca535ccaa9c5fe3b
Parents: a382034
Author: Julien Le Dem <ju...@apache.org>
Authored: Thu Jun 8 16:40:41 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
.../org/apache/arrow/vector/stream/MessageSerializer.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/a44d5842/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
index 2fd9374..05ad92d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java
@@ -142,7 +142,7 @@ public class MessageSerializer {
return new ArrowBlock(start, metadataLength + 4, bufferLength);
}
- private static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) throws IOException {
+ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) throws IOException {
long bufferStart = out.getCurrentPosition();
List<ArrowBuf> buffers = batch.getBuffers();
List<ArrowBuffer> buffersLayout = batch.getBuffersLayout();
@@ -171,7 +171,7 @@ public class MessageSerializer {
* @return the deserialized object
* @throws IOException if something went wrong
*/
- private static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
+ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc)
throws IOException {
RecordBatch recordBatchFB = (RecordBatch) message.header(new RecordBatch());
@@ -222,7 +222,7 @@ public class MessageSerializer {
}
// Deserializes a record batch given the Flatbuffer metadata and in-memory body
- private static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
+ public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB,
ArrowBuf body) throws IOException {
// Now read the body
int nodesLength = recordBatchFB.nodesLength();
@@ -296,7 +296,7 @@ public class MessageSerializer {
* @return the corresponding dictionary batch
* @throws IOException if something went wrong
*/
- private static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
+ public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in,
Message message,
BufferAllocator alloc) throws IOException {
DictionaryBatch dictionaryBatchFB = (DictionaryBatch) message.header(new DictionaryBatch());
@@ -373,7 +373,7 @@ public class MessageSerializer {
* @param bodyLength body length field
* @return the corresponding ByteBuffer
*/
- private static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
+ public static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte headerType,
int headerOffset, int bodyLength) {
Message.startMessage(builder);
Message.addHeaderType(builder, headerType);
[12/12] arrow git commit: ARROW-1109: [JAVA] transferOwnership fails
when readerIndex is not 0
Posted by we...@apache.org.
ARROW-1109: [JAVA] transferOwnership fails when readerIndex is not 0
Closes #745
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/0e680f01
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/0e680f01
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/0e680f01
Branch: refs/heads/master
Commit: 0e680f01c8035ded17496a369cafebfec9968392
Parents: 06c26a2
Author: Julien Le Dem <ju...@apache.org>
Authored: Tue Apr 11 09:40:09 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
.../src/main/java/io/netty/buffer/ArrowBuf.java | 29 ++++++++++----------
1 file changed, 14 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/0e680f01/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
----------------------------------------------------------------------
diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 06272a7..0328a16 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -18,19 +18,6 @@
package io.netty.buffer;
-import com.google.common.base.Preconditions;
-
-import io.netty.util.internal.PlatformDependent;
-
-import org.apache.arrow.memory.AllocationManager.BufferLedger;
-import org.apache.arrow.memory.ArrowByteBufAllocator;
-import org.apache.arrow.memory.BaseAllocator;
-import org.apache.arrow.memory.BaseAllocator.Verbosity;
-import org.apache.arrow.memory.BoundsChecking;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.BufferManager;
-import org.apache.arrow.memory.util.HistoricalLog;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -42,6 +29,19 @@ import java.nio.charset.Charset;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.arrow.memory.AllocationManager.BufferLedger;
+import org.apache.arrow.memory.ArrowByteBufAllocator;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BaseAllocator.Verbosity;
+import org.apache.arrow.memory.BoundsChecking;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.BufferManager;
+import org.apache.arrow.memory.util.HistoricalLog;
+
+import com.google.common.base.Preconditions;
+
+import io.netty.util.internal.PlatformDependent;
+
public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ArrowBuf.class);
@@ -245,8 +245,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null);
- newBuf.readerIndex(this.readerIndex);
- newBuf.writerIndex(this.writerIndex);
+ newBuf.setIndex(this.readerIndex, this.writerIndex);
final boolean allocationFit = this.ledger.transferBalance(otherLedger);
return new TransferResult(allocationFit, newBuf);
}
[02/12] arrow git commit: [maven-release-plugin] prepare for next
development iteration
Posted by we...@apache.org.
[maven-release-plugin] prepare for next development iteration
Change-Id: Iec07f0f0117c363e39666a76c5c99e7f94c73ab0
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/41b58e4f
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/41b58e4f
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/41b58e4f
Branch: refs/heads/master
Commit: 41b58e4f7c6ad2f1aa0d0364349b6da30480349e
Parents: 4631543
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue Jun 6 22:13:36 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue Jun 6 22:13:36 2017 -0400
----------------------------------------------------------------------
java/format/pom.xml | 2 +-
java/memory/pom.xml | 2 +-
java/pom.xml | 4 ++--
java/tools/pom.xml | 2 +-
java/vector/pom.xml | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/41b58e4f/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
index 9d509bc..e092754 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>arrow-java-root</artifactId>
<groupId>org.apache.arrow</groupId>
- <version>0.4.1</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-format</artifactId>
http://git-wip-us.apache.org/repos/asf/arrow/blob/41b58e4f/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 6a8fab0..dc4d0da 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.1</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>Arrow Memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/41b58e4f/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 094914b..1ec3d56 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.1</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
<url>https://github.com/apache/arrow</url>
- <tag>apache-arrow-0.4.1</tag>
+ <tag>HEAD</tag>
</scm>
<mailingLists>
http://git-wip-us.apache.org/repos/asf/arrow/blob/41b58e4f/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index f93cef8..53331f2 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.1</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-tools</artifactId>
<name>Arrow Tools</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/41b58e4f/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 60e342d..a117a2f 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.1</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-vector</artifactId>
<name>Arrow Vectors</name>
[08/12] arrow git commit: ARROW-1110: [JAVA] make union vector naming
consistent
Posted by we...@apache.org.
ARROW-1110: [JAVA] make union vector naming consistent
Closes #746
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/c6cf1244
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/c6cf1244
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/c6cf1244
Branch: refs/heads/master
Commit: c6cf1244227f9043561463116e24abd1d5713a85
Parents: 0e680f0
Author: Julien Le Dem <ju...@apache.org>
Authored: Fri Apr 14 10:23:21 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/arrow/vector/complex/ListVector.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/c6cf1244/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index c2f5576..b5a27b5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -311,7 +311,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
@Override
public UnionVector promoteToUnion() {
- UnionVector vector = new UnionVector(name, allocator, callBack);
+ UnionVector vector = new UnionVector("$data$", allocator, callBack);
replaceDataVector(vector);
reader = new UnionListReader(this);
if (callBack != null) {
[11/12] arrow git commit: ARROW-1112: [JAVA] Set lastSet for
VarLength and List vectors when loading
Posted by we...@apache.org.
ARROW-1112: [JAVA] Set lastSet for VarLength and List vectors when loading
Closes #748
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/ac648539
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/ac648539
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/ac648539
Branch: refs/heads/master
Commit: ac6485390b04786959a42d55cd28d64bfbb40589
Parents: 11deee6
Author: Steven Phillips <st...@dremio.com>
Authored: Tue Jun 6 15:19:42 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
java/vector/src/main/codegen/templates/NullableValueVectors.java | 1 +
.../src/main/java/org/apache/arrow/vector/complex/ListVector.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/ac648539/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 4aa0076..092097b 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -143,6 +143,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1,
values.offsetVector.getBufferSizeFor(
fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1));
+ mutator.lastSet = fieldNode.getLength() - 1;
<#else>
// fixed width values truncate value vector to size (#1)
org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength()));
http://git-wip-us.apache.org/repos/asf/arrow/blob/ac648539/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index b5a27b5..4ab624f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -115,6 +115,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
// variable width values: truncate offset vector buffer to size (#1)
org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1));
BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
+ lastSet = fieldNode.getLength();
}
@Override
[05/12] arrow git commit: ARROW-1108: [JAVA] Check if ArrowBuf is
empty buffer in getActualConsumedMemory() and getPossibleConsumedMemory()
Posted by we...@apache.org.
ARROW-1108: [JAVA] Check if ArrowBuf is empty buffer in getActualConsumedMemory() and getPossibleConsumedMemory()
Closes #744
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/06c26a2c
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/06c26a2c
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/06c26a2c
Branch: refs/heads/master
Commit: 06c26a2cd6d361a2df84e5ab4146c83513d3ee46
Parents: ae6142d
Author: Steven Phillips <st...@dremio.com>
Authored: Mon Mar 27 14:24:01 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
java/memory/src/main/java/io/netty/buffer/ArrowBuf.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/06c26a2c/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
----------------------------------------------------------------------
diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index e777b5a..06272a7 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -796,6 +796,9 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
* @return Size in bytes.
*/
public int getPossibleMemoryConsumed() {
+ if (isEmpty) {
+ return 0;
+ }
return ledger.getSize();
}
@@ -807,6 +810,9 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
* @return Size in bytes.
*/
public int getActualMemoryConsumed() {
+ if (isEmpty) {
+ return 0;
+ }
return ledger.getAccountedSize();
}
[04/12] arrow git commit: ARROW-1107: [JAVA] Fix getField() for
NullableMapVector
Posted by we...@apache.org.
ARROW-1107: [JAVA] Fix getField() for NullableMapVector
Closes #743
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/ae6142da
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/ae6142da
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/ae6142da
Branch: refs/heads/master
Commit: ae6142da6171c9d61c2e2704af32b4da42b68f11
Parents: a44d584
Author: Steven Phillips <st...@dremio.com>
Authored: Thu Feb 2 18:53:54 2017 -0800
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
.../org/apache/arrow/vector/complex/NullableMapVector.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/ae6142da/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index f1f8640..ee95fde 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -41,6 +41,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;
@@ -82,6 +83,12 @@ public class NullableMapVector extends MapVector implements FieldVector {
}
@Override
+ public Field getField() {
+ Field f = super.getField();
+ return new Field(f.getName(), true, f.getType(), f.getChildren());
+ }
+
+ @Override
public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers);
this.valueCount = fieldNode.getLength();
[07/12] arrow git commit: ARROW-1048: Use existing LD_LIBRARY_PATH in
source release script to accommodate non-system toolchain libs
Posted by we...@apache.org.
ARROW-1048: Use existing LD_LIBRARY_PATH in source release script to accommodate non-system toolchain libs
Tripped on this again when cutting the 0.4.1 rc0. cc @kou
Author: Wes McKinney <we...@twosigma.com>
Closes #738 from wesm/ARROW-1048 and squashes the following commits:
f335c09 [Wes McKinney] Use existing LD_LIBRARY_PATH in source release script to accommodate non-system toolchain libraries
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/7a7b0c22
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/7a7b0c22
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/7a7b0c22
Branch: refs/heads/master
Commit: 7a7b0c22ad28e44a2a83f63ef521c53f489dd8dc
Parents: e344066
Author: Wes McKinney <we...@twosigma.com>
Authored: Wed Jun 7 22:31:39 2017 +0900
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
dev/release/02-source.sh | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/7a7b0c22/dev/release/02-source.sh
----------------------------------------------------------------------
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index d3d94af..508141e 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -74,7 +74,7 @@ cd ${extract_dir}/c_glib
./configure \
PKG_CONFIG_PATH=$cpp_install_dir/lib/pkgconfig \
--enable-gtk-doc
-LD_LIBRARY_PATH=$cpp_install_dir/lib make -j8
+LD_LIBRARY_PATH=$cpp_install_dir/lib:$LD_LIBRARY_PATH make -j8
make dist
tar xzf *.tar.gz
rm *.tar.gz
@@ -114,4 +114,3 @@ echo "Success! The release candidate is available here:"
echo " https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
echo ""
echo "Commit SHA1: ${release_hash}"
-
[10/12] arrow git commit: ARROW-1111: [JAVA] Make aligning buffers
optional, and allow -1 for unknown null count
Posted by we...@apache.org.
ARROW-1111: [JAVA] Make aligning buffers optional, and allow -1 for unknown null count
Closes #747
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/11deee61
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/11deee61
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/11deee61
Branch: refs/heads/master
Commit: 11deee61aa2fd6782df1f8eefa15938a7676a546
Parents: c6cf124
Author: Steven Phillips <st...@dremio.com>
Authored: Tue Jun 6 15:19:00 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/arrow/vector/VectorUnloader.java | 12 ++++++++++--
.../apache/arrow/vector/schema/ArrowRecordBatch.java | 8 ++++++--
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/11deee61/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 8e9ff6d..9965245 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -29,9 +29,17 @@ import org.apache.arrow.vector.schema.ArrowVectorType;
public class VectorUnloader {
private final VectorSchemaRoot root;
+ private final boolean includeNullCount;
+ private final boolean alignBuffers;
public VectorUnloader(VectorSchemaRoot root) {
+ this(root, true, true);
+ }
+
+ public VectorUnloader(VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers) {
this.root = root;
+ this.includeNullCount = includeNullCount;
+ this.alignBuffers = alignBuffers;
}
public ArrowRecordBatch getRecordBatch() {
@@ -40,12 +48,12 @@ public class VectorUnloader {
for (FieldVector vector : root.getFieldVectors()) {
appendNodes(vector, nodes, buffers);
}
- return new ArrowRecordBatch(root.getRowCount(), nodes, buffers);
+ return new ArrowRecordBatch(root.getRowCount(), nodes, buffers, alignBuffers);
}
private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
Accessor accessor = vector.getAccessor();
- nodes.add(new ArrowFieldNode(accessor.getValueCount(), accessor.getNullCount()));
+ nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1));
List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
if (fieldBuffers.size() != expectedBuffers.size()) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/11deee61/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 6ef514e..6a716fa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -48,12 +48,16 @@ public class ArrowRecordBatch implements ArrowMessage {
private boolean closed = false;
+ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+ this(length, nodes, buffers, true);
+ }
+
/**
* @param length how many rows in this batch
* @param nodes field level info
* @param buffers will be retained until this recordBatch is closed
*/
- public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+ public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers, boolean alignBuffers) {
super();
this.length = length;
this.nodes = nodes;
@@ -66,7 +70,7 @@ public class ArrowRecordBatch implements ArrowMessage {
arrowBuffers.add(new ArrowBuffer(0, offset, size));
LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size));
offset += size;
- if (offset % 8 != 0) { // align on 8 byte boundaries
+ if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries
offset += 8 - (offset % 8);
}
}
[06/12] arrow git commit: ARROW-1095: Add Arrow logo PNG to website
img folder
Posted by we...@apache.org.
ARROW-1095: Add Arrow logo PNG to website img folder
Author: Wes McKinney <we...@twosigma.com>
Closes #739 from wesm/ARROW-1095 and squashes the following commits:
4a89d2d [Wes McKinney] Add Arrow logo PNG
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e344066c
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e344066c
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e344066c
Branch: refs/heads/master
Commit: e344066c46716b9144cc385a9d6546a09e2d4ae4
Parents: 41b58e4
Author: Wes McKinney <we...@twosigma.com>
Authored: Wed Jun 7 21:26:20 2017 +0900
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
site/img/arrow.png | Bin 0 -> 21475 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/e344066c/site/img/arrow.png
----------------------------------------------------------------------
diff --git a/site/img/arrow.png b/site/img/arrow.png
new file mode 100644
index 0000000..20b00bd
Binary files /dev/null and b/site/img/arrow.png differ
[09/12] arrow git commit: ARROW-1101: Implement write(TypeHolder)
methods in UnionListWriter
Posted by we...@apache.org.
ARROW-1101: Implement write(TypeHolder) methods in UnionListWriter
Author: vkorukanti <ve...@dremio.com>
Closes #740 from vkorukanti/ARROW-1101 and squashes the following commits:
e2d097d [vkorukanti] ARROW-1101: Implement write(TypeHolder) methods in UnionListWriter
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a3820346
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a3820346
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a3820346
Branch: refs/heads/master
Commit: a38203469530fb0ce56be71d94a6848085ab2c1c
Parents: 7a7b0c2
Author: vkorukanti <ve...@dremio.com>
Authored: Wed Jun 7 17:01:30 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400
----------------------------------------------------------------------
java/vector/src/main/codegen/templates/UnionListWriter.java | 5 +++++
.../arrow/vector/complex/writer/TestComplexWriter.java | 9 ++++++++-
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/a3820346/java/vector/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index d6e2f9f..9fe41d0 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -166,6 +166,11 @@ public class UnionListWriter extends AbstractFieldWriter {
writer.setPosition(writer.idx()+1);
}
+ public void write(${name}Holder holder) {
+ writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
+ writer.setPosition(writer.idx()+1);
+ }
+
</#if>
</#list>
</#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/a3820346/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index ede8d65..5a9c80d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -42,6 +42,7 @@ import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.holders.IntHolder;
import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
@@ -216,7 +217,13 @@ public class TestComplexWriter {
for (int i = 0; i < COUNT; i++) {
listWriter.startList();
for (int j = 0; j < i % 7; j++) {
- listWriter.writeInt(j);
+ if (j%2 == 0) {
+ listWriter.writeInt(j);
+ } else {
+ IntHolder holder = new IntHolder();
+ holder.value = j;
+ listWriter.write(holder);
+ }
}
listWriter.endList();
}