You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2018/05/29 22:29:50 UTC
[arrow] branch master updated: ARROW-2632: [Java] ArrowStreamWriter
accumulates ArrowBlock but does not use them
This is an automated email from the ASF dual-hosted git repository.
cutlerb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d8bce06 ARROW-2632: [Java] ArrowStreamWriter accumulates ArrowBlock but does not use them
d8bce06 is described below
commit d8bce0660ee34eea819e56e750a5a3437b0a9ff2
Author: Teddy Choi <pu...@gmail.com>
AuthorDate: Tue May 29 15:29:35 2018 -0700
ARROW-2632: [Java] ArrowStreamWriter accumulates ArrowBlock but does not use them
ArrowStreamWriter inherits from ArrowWriter the behavior of accumulating an ArrowBlock for each ArrowRecordBatch that is written. But this data is never used in the context of ArrowStreamWriter. writeRecordBatch can be overridden in ArrowStreamWriter to avoid adding the memory overhead.
Author: Teddy Choi <pu...@gmail.com>
Closes #2079 from pudidic/ARROW-2632 and squashes the following commits:
70ecf08 <Teddy Choi> Preserve logging. Add more list size checks in unit tests.
7b289d4 <Teddy Choi> ARROW-2632: ArrowStreamWriter accumulates ArrowBlock but does not use them
---
.../src/test/java/org/apache/arrow/tools/EchoServerTest.java | 1 +
.../java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java | 9 +++++++++
.../src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java | 10 ++++++++--
.../test/java/org/apache/arrow/vector/ipc/TestArrowStream.java | 4 ++++
4 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 47b5541..2674c7b 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -112,6 +112,7 @@ public class EchoServerTest {
writer.writeBatch();
}
writer.end();
+ assertTrue(writer.getRecordBlocks().isEmpty());
assertEquals(new Schema(asList(field)), reader.getVectorSchemaRoot().getSchema());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
index d731d05..14e6add 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
@@ -23,6 +23,8 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.ArrowWriter;
import org.apache.arrow.vector.ipc.WriteChannel;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Schema;
import java.io.IOException;
@@ -52,4 +54,11 @@ public class ArrowStreamWriter extends ArrowWriter {
List<ArrowBlock> records) throws IOException {
out.writeIntLittleEndian(0);
}
+
+ @Override
+ protected void writeRecordBatch(ArrowRecordBatch batch) throws IOException {
+ ArrowBlock block = MessageSerializer.serialize(out, batch);
+ LOGGER.debug(String.format("RecordBatch at %d, metadata: %d, body: %d",
+ block.getOffset(), block.getMetadataLength(), block.getBodyLength()));
+ }
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
index 4b483d0..d9d0534 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.VectorUnloader;
@@ -44,11 +45,11 @@ import com.google.common.collect.ImmutableList;
public abstract class ArrowWriter implements AutoCloseable {
- private static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
+ protected static final Logger LOGGER = LoggerFactory.getLogger(ArrowWriter.class);
// schema with fields in message format, not memory format
private final Schema schema;
- private final WriteChannel out;
+ protected final WriteChannel out;
private final VectorUnloader unloader;
private final List<ArrowDictionaryBatch> dictionaries;
@@ -164,4 +165,9 @@ public abstract class ArrowWriter implements AutoCloseable {
throw new RuntimeException(e);
}
}
+
+ @VisibleForTesting
+ public List<ArrowBlock> getRecordBlocks() {
+ return recordBlocks;
+ }
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
index 431ebf0..bed4e63 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java
@@ -82,6 +82,8 @@ public class TestArrowStream extends BaseFileTest {
}
writer.end();
bytesWritten = writer.bytesWritten();
+
+ assertTrue(writer.getRecordBlocks().isEmpty());
}
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
@@ -108,6 +110,8 @@ public class TestArrowStream extends BaseFileTest {
try (VectorSchemaRoot root = new VectorSchemaRoot(schema, Collections.singletonList(vector), vector.getValueCount());
ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(os));) {
writeBatchData(writer, vector, root);
+
+ assertTrue(writer.getRecordBlocks().isEmpty());
}
}
--
To stop receiving notification emails like this one, please contact
cutlerb@apache.org.