You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ga...@apache.org on 2021/01/05 10:50:26 UTC
[parquet-mr] branch master updated: PARQUET-1954: TCP connection
leak in parquet dump (#849)
This is an automated email from the ASF dual-hosted git repository.
gabor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 74af3a8 PARQUET-1954: TCP connection leak in parquet dump (#849)
74af3a8 is described below
commit 74af3a8a5404e3a56d3d8d6bf3ebc8c09ff5fa1d
Author: Jeff.r <st...@126.com>
AuthorDate: Tue Jan 5 18:50:16 2021 +0800
PARQUET-1954: TCP connection leak in parquet dump (#849)
---
.../apache/parquet/tools/command/DumpCommand.java | 91 ++++++++++------------
1 file changed, 41 insertions(+), 50 deletions(-)
diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/command/DumpCommand.java b/parquet-tools/src/main/java/org/apache/parquet/tools/command/DumpCommand.java
index eaf6e8e..6ae31c5 100644
--- a/parquet-tools/src/main/java/org/apache/parquet/tools/command/DumpCommand.java
+++ b/parquet-tools/src/main/java/org/apache/parquet/tools/command/DumpCommand.java
@@ -163,50 +163,44 @@ public class DumpCommand extends ArgsOnlyCommand {
}
}
- ParquetFileReader freader = null;
if (showmd) {
- try {
- long group = 0;
- for (BlockMetaData block : blocks) {
- if (group != 0) out.println();
- out.format("row group %d%n", group++);
- out.rule('-');
-
-
- List<ColumnChunkMetaData> ccmds = block.getColumns();
- if (showColumns != null) {
- ccmds = new ArrayList<ColumnChunkMetaData>();
- for (ColumnChunkMetaData ccmd : block.getColumns()) {
- String path = Joiner.on('.').skipNulls().join(ccmd.getPath().toArray());
- if (showColumns.contains(path)) {
- ccmds.add(ccmd);
- }
- }
- }
-
- MetadataUtils.showDetails(out, ccmds);
-
- List<BlockMetaData> rblocks = Collections.singletonList(block);
- freader = new ParquetFileReader(
- conf, meta.getFileMetaData(), inpath, rblocks, columns);
- PageReadStore store = freader.readNextRowGroup();
- while (store != null) {
- out.incrementTabLevel();
- for (ColumnDescriptor column : columns) {
- out.println();
- dump(out, store, column);
- }
- out.decrementTabLevel();
-
- store = freader.readNextRowGroup();
- }
- out.flushColumns();
+ long group = 0;
+ for (BlockMetaData block : blocks) {
+ if (group != 0)
+ out.println();
+ out.format("row group %d%n", group++);
+ out.rule('-');
+
+ List<ColumnChunkMetaData> ccmds = block.getColumns();
+ if (showColumns != null) {
+ ccmds = new ArrayList<ColumnChunkMetaData>();
+ for (ColumnChunkMetaData ccmd : block.getColumns()) {
+ String path = Joiner.on('.').skipNulls().join(ccmd.getPath().toArray());
+ if (showColumns.contains(path)) {
+ ccmds.add(ccmd);
}
- } finally {
- if (freader != null) {
- freader.close();
+ }
+ }
+
+ MetadataUtils.showDetails(out, ccmds);
+
+ List<BlockMetaData> rblocks = Collections.singletonList(block);
+ try (ParquetFileReader freader = new ParquetFileReader(conf, meta.getFileMetaData(), inpath, rblocks,
+ columns)) {
+ PageReadStore store = freader.readNextRowGroup();
+ while (store != null) {
+ out.incrementTabLevel();
+ for (ColumnDescriptor column : columns) {
+ out.println();
+ dump(out, store, column);
}
+ out.decrementTabLevel();
+
+ store = freader.readNextRowGroup();
+ }
+ out.flushColumns();
}
+ }
}
if (showdt) {
@@ -221,25 +215,22 @@ public class DumpCommand extends ArgsOnlyCommand {
long page = 1;
long total = blocks.size();
long offset = 1;
- freader = new ParquetFileReader(
- conf, meta.getFileMetaData(), inpath, blocks, Collections.singletonList(column));
- PageReadStore store = freader.readNextRowGroup();
- while (store != null) {
+ try(ParquetFileReader freader = new ParquetFileReader(
+ conf, meta.getFileMetaData(), inpath, blocks, Collections.singletonList(column))){
+ PageReadStore store = freader.readNextRowGroup();
+ while (store != null) {
ColumnReadStoreImpl crstore = new ColumnReadStoreImpl(
- store, new DumpGroupConverter(), schema,
- meta.getFileMetaData().getCreatedBy());
+ store, new DumpGroupConverter(), schema,
+ meta.getFileMetaData().getCreatedBy());
dump(out, crstore, column, page++, total, offset);
offset += store.getRowCount();
store = freader.readNextRowGroup();
+ }
}
-
out.flushColumns();
} finally {
out.flushColumns();
- if (freader != null) {
- freader.close();
- }
}
}
}