You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ga...@apache.org on 2019/07/09 08:27:53 UTC
[parquet-mr] branch master updated: PARQUET-1615: getRecordWriter
shouldn't hardcode CREAT mode when new ParquetFileWriter (#660)
This is an automated email from the ASF dual-hosted git repository.
gabor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 8ff867a PARQUET-1615: getRecordWriter shouldn't hardcode CREAT mode when new ParquetFileWriter (#660)
8ff867a is described below
commit 8ff867a2e183f50b7b2f2c6e51d07c5314577ce0
Author: Alan Jin <La...@users.noreply.github.com>
AuthorDate: Tue Jul 9 16:27:48 2019 +0800
PARQUET-1615: getRecordWriter shouldn't hardcode CREAT mode when new ParquetFileWriter (#660)
---
.../apache/parquet/hadoop/ParquetOutputFormat.java | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
index 04cbd15..cd25b23 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
@@ -366,21 +366,36 @@ public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> {
@Override
public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext)
throws IOException, InterruptedException {
+ return getRecordWriter(taskAttemptContext, Mode.CREATE);
+ }
+
+ public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext, Mode mode)
+ throws IOException, InterruptedException {
final Configuration conf = getConfiguration(taskAttemptContext);
CompressionCodecName codec = getCodec(taskAttemptContext);
String extension = codec.getExtension() + ".parquet";
Path file = getDefaultWorkFile(taskAttemptContext, extension);
- return getRecordWriter(conf, file, codec);
+ return getRecordWriter(conf, file, codec, mode);
}
public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext, Path file)
+ throws IOException, InterruptedException {
+ return getRecordWriter(taskAttemptContext, file, Mode.CREATE);
+ }
+
+ public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext, Path file, Mode mode)
throws IOException, InterruptedException {
- return getRecordWriter(getConfiguration(taskAttemptContext), file, getCodec(taskAttemptContext));
+ return getRecordWriter(getConfiguration(taskAttemptContext), file, getCodec(taskAttemptContext), mode);
}
public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, CompressionCodecName codec)
+ throws IOException, InterruptedException {
+ return getRecordWriter(conf, file, codec, Mode.CREATE);
+ }
+
+ public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, CompressionCodecName codec, Mode mode)
throws IOException, InterruptedException {
final WriteSupport<T> writeSupport = getWriteSupport(conf);
@@ -417,7 +432,7 @@ public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> {
WriteContext init = writeSupport.init(conf);
ParquetFileWriter w = new ParquetFileWriter(HadoopOutputFile.fromPath(file, conf),
- init.getSchema(), Mode.CREATE, blockSize, maxPaddingSize, props.getColumnIndexTruncateLength());
+ init.getSchema(), mode, blockSize, maxPaddingSize, props.getColumnIndexTruncateLength());
w.start();
float maxLoad = conf.getFloat(ParquetOutputFormat.MEMORY_POOL_RATIO,