You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ga...@apache.org on 2024/03/20 05:47:34 UTC
(parquet-mr) branch master updated: PARQUET-2449: Limit local buffer to be similar to Hadoop writing (#1299)
This is an automated email from the ASF dual-hosted git repository.
gangwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new acab1e83f PARQUET-2449: Limit local buffer to be similar to Hadoop writing (#1299)
acab1e83f is described below
commit acab1e83f41143b99d4f0eac04c05935aa935c7c
Author: Atour <28...@users.noreply.github.com>
AuthorDate: Wed Mar 20 06:47:29 2024 +0100
PARQUET-2449: Limit local buffer to be similar to Hadoop writing (#1299)
---
.../main/java/org/apache/parquet/io/LocalOutputFile.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/parquet-common/src/main/java/org/apache/parquet/io/LocalOutputFile.java b/parquet-common/src/main/java/org/apache/parquet/io/LocalOutputFile.java
index 785844dd5..9c823f92e 100644
--- a/parquet-common/src/main/java/org/apache/parquet/io/LocalOutputFile.java
+++ b/parquet-common/src/main/java/org/apache/parquet/io/LocalOutputFile.java
@@ -30,6 +30,8 @@ import java.nio.file.StandardOpenOption;
*/
public class LocalOutputFile implements OutputFile {
+ private static final int BUFFER_SIZE_DEFAULT = 4096;
+
private class LocalPositionOutputStream extends PositionOutputStream {
private final BufferedOutputStream stream;
@@ -80,24 +82,24 @@ public class LocalOutputFile implements OutputFile {
}
@Override
- public PositionOutputStream create(long buffer) throws IOException {
- return new LocalPositionOutputStream((int) buffer, StandardOpenOption.CREATE_NEW);
+ public PositionOutputStream create(long blockSize) throws IOException {
+ return new LocalPositionOutputStream(BUFFER_SIZE_DEFAULT, StandardOpenOption.CREATE_NEW);
}
@Override
- public PositionOutputStream createOrOverwrite(long buffer) throws IOException {
+ public PositionOutputStream createOrOverwrite(long blockSize) throws IOException {
return new LocalPositionOutputStream(
- (int) buffer, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+ BUFFER_SIZE_DEFAULT, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
}
@Override
public boolean supportsBlockSize() {
- return true;
+ return false;
}
@Override
public long defaultBlockSize() {
- return 512;
+ return -1;
}
@Override