You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2017/01/19 01:27:44 UTC
[33/50] [abbrv] parquet-mr git commit: PARQUET-544: Add closed flag
to allow for closeable contract adherence
PARQUET-544: Add closed flag to allow for closeable contract adherence
The closeable interface states:
> Closes this stream and releases any system resources associated with it. If the stream is already closed then invoking this method has no effect.
As InternalParquetRecordWriter implements this interface we should adhere to this contract.
Author: Mark Reddy <ma...@gmail.com>
Closes #345 from markreddy/PARQUET-544-adhere-to-closeable-contract and squashes the following commits:
135db9b [Mark Reddy] PARQUET-544: add closed flag to allow for adherence to closeable contract
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/4240cb97
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/4240cb97
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/4240cb97
Branch: refs/heads/parquet-1.8.x
Commit: 4240cb975a3a8a76b27fc2f25faa1d791220a20d
Parents: e3ee35a
Author: Mark Reddy <ma...@gmail.com>
Authored: Thu Jun 30 09:41:51 2016 -0700
Committer: Ryan Blue <bl...@apache.org>
Committed: Mon Jan 9 16:54:54 2017 -0800
----------------------------------------------------------------------
.../hadoop/InternalParquetRecordWriter.java | 21 ++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/4240cb97/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
index 26d9c50..d5c57ab 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
@@ -56,6 +56,8 @@ class InternalParquetRecordWriter<T> {
private final boolean validating;
private final ParquetProperties props;
+ private boolean closed;
+
private long recordCount = 0;
private long recordCountForNextMemCheck = MINIMUM_RECORD_COUNT_FOR_CHECK;
private long lastRowGroupEndPos = 0;
@@ -103,15 +105,18 @@ class InternalParquetRecordWriter<T> {
}
public void close() throws IOException, InterruptedException {
- flushRowGroupToStore();
- FinalizedWriteContext finalWriteContext = writeSupport.finalizeWrite();
- Map<String, String> finalMetadata = new HashMap<String, String>(extraMetaData);
- String modelName = writeSupport.getName();
- if (modelName != null) {
- finalMetadata.put(ParquetWriter.OBJECT_MODEL_NAME_PROP, modelName);
+ if (!closed) {
+ flushRowGroupToStore();
+ FinalizedWriteContext finalWriteContext = writeSupport.finalizeWrite();
+ Map<String, String> finalMetadata = new HashMap<String, String>(extraMetaData);
+ String modelName = writeSupport.getName();
+ if (modelName != null) {
+ finalMetadata.put(ParquetWriter.OBJECT_MODEL_NAME_PROP, modelName);
+ }
+ finalMetadata.putAll(finalWriteContext.getExtraMetaData());
+ parquetFileWriter.end(finalMetadata);
+ closed = true;
}
- finalMetadata.putAll(finalWriteContext.getExtraMetaData());
- parquetFileWriter.end(finalMetadata);
}
public void write(T value) throws IOException, InterruptedException {