You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by we...@apache.org on 2016/09/01 20:30:44 UTC
parquet-cpp git commit: PARQUET-701: Ensure that Close can be called
multiple times
Repository: parquet-cpp
Updated Branches:
refs/heads/master bf6716c76 -> 261072ca9
PARQUET-701: Ensure that Close can be called multiple times
Author: Uwe L. Korn <uw...@xhochy.com>
Closes #149 from xhochy/parquet-701 and squashes the following commits:
6f29d8b [Uwe L. Korn] PARQUET-701: Ensure that Close can be called multiple times
Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/261072ca
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/261072ca
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/261072ca
Branch: refs/heads/master
Commit: 261072ca966bfb17e2ebac232e8f9c2d62f4618d
Parents: bf6716c
Author: Uwe L. Korn <uw...@xhochy.com>
Authored: Thu Sep 1 16:30:36 2016 -0400
Committer: Wes McKinney <we...@apache.org>
Committed: Thu Sep 1 16:30:36 2016 -0400
----------------------------------------------------------------------
src/parquet/column/column-writer-test.cc | 2 ++
src/parquet/column/writer.cc | 20 +++++++++++++-------
src/parquet/column/writer.h | 1 +
3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/261072ca/src/parquet/column/column-writer-test.cc
----------------------------------------------------------------------
diff --git a/src/parquet/column/column-writer-test.cc b/src/parquet/column/column-writer-test.cc
index f2c9c64..ab232ea 100644
--- a/src/parquet/column/column-writer-test.cc
+++ b/src/parquet/column/column-writer-test.cc
@@ -109,6 +109,8 @@ class TestPrimitiveWriter : public ::testing::Test {
std::unique_ptr<TypedColumnWriter<TestType>> writer =
this->BuildWriter(SMALL_SIZE, encoding);
writer->WriteBatch(this->values_.size(), nullptr, nullptr, this->values_ptr_);
+ // The behaviour should be independent from the number of Close() calls
+ writer->Close();
writer->Close();
this->ReadColumn();
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/261072ca/src/parquet/column/writer.cc
----------------------------------------------------------------------
diff --git a/src/parquet/column/writer.cc b/src/parquet/column/writer.cc
index 124486c..1c376ad 100644
--- a/src/parquet/column/writer.cc
+++ b/src/parquet/column/writer.cc
@@ -46,7 +46,8 @@ ColumnWriter::ColumnWriter(const ColumnDescriptor* descr,
num_buffered_values_(0),
num_buffered_encoded_values_(0),
num_rows_(0),
- total_bytes_written_(0) {
+ total_bytes_written_(0),
+ closed_(false) {
InitSinks();
}
@@ -56,11 +57,13 @@ void ColumnWriter::InitSinks() {
}
void ColumnWriter::WriteDefinitionLevels(int64_t num_levels, const int16_t* levels) {
+ DCHECK(!closed_);
definition_levels_sink_->Write(
reinterpret_cast<const uint8_t*>(levels), sizeof(int16_t) * num_levels);
}
void ColumnWriter::WriteRepetitionLevels(int64_t num_levels, const int16_t* levels) {
+ DCHECK(!closed_);
repetition_levels_sink_->Write(
reinterpret_cast<const uint8_t*>(levels), sizeof(int16_t) * num_levels);
}
@@ -129,12 +132,15 @@ void ColumnWriter::WriteDataPage(const DataPage& page) {
}
int64_t ColumnWriter::Close() {
- if (has_dictionary_) { WriteDictionaryPage(); }
- // Write all outstanding data to a new page
- if (num_buffered_values_ > 0) { AddDataPage(); }
-
- for (size_t i = 0; i < data_pages_.size(); i++) {
- WriteDataPage(data_pages_[i]);
+ if (!closed_) {
+ closed_ = true;
+ if (has_dictionary_) { WriteDictionaryPage(); }
+ // Write all outstanding data to a new page
+ if (num_buffered_values_ > 0) { AddDataPage(); }
+
+ for (size_t i = 0; i < data_pages_.size(); i++) {
+ WriteDataPage(data_pages_[i]);
+ }
}
if (num_rows_ != expected_rows_) {
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/261072ca/src/parquet/column/writer.h
----------------------------------------------------------------------
diff --git a/src/parquet/column/writer.h b/src/parquet/column/writer.h
index f7834f5..c88ead1 100644
--- a/src/parquet/column/writer.h
+++ b/src/parquet/column/writer.h
@@ -101,6 +101,7 @@ class PARQUET_EXPORT ColumnWriter {
int num_rows_;
int total_bytes_written_;
+ bool closed_;
std::unique_ptr<InMemoryOutputStream> definition_levels_sink_;
std::unique_ptr<InMemoryOutputStream> repetition_levels_sink_;