You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2021/08/11 16:22:09 UTC
[orc] branch main updated: ORC-937: Replace deprecated method (#850)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new 4799fa5 ORC-937: Replace deprecated method (#850)
4799fa5 is described below
commit 4799fa56f3c6fa33a8e72f3106b63d097c0f2845
Author: guiyanakaung <gu...@gmail.com>
AuthorDate: Thu Aug 12 00:22:05 2021 +0800
ORC-937: Replace deprecated method (#850)
### What changes were proposed in this pull request?
Exclude calls to deprecated methods for compatibility or active testing. Replace all deprecated methods with recommended methods.
org.apache.commons.cli
OptionBuilder -> Option.builder
GnuParser -> DefaultParser
org.apache.parquet.avro
AvroParquetWriter.<GenericData.Record>builder(path) -> AvroParquetWriter.<GenericData.Record>builder(outputFile)
AvroParquetReader.<GenericData.Record>builder(path) -> AvroParquetReader.<GenericData.Record>builder(inputFile)
DateColumnStatistics
getMaximum -> getMaximumLocalDate
getMinimum -> getMinimumLocalDate
hadoop
FSDataOutputStream(OutputStream) -> FSDataOutputStream(OutputStream, FileSystem.Statistics)
fileStatus.isDir -> fileStatus.isDirectory
### Why are the changes needed?
Use the recommended method. Possible deprecated methods are removed when dependencies are upgraded.
### How was this patch tested?
Pass the CIs
---
.../bench/core/convert/parquet/ParquetReader.java | 4 +-
.../bench/core/convert/parquet/ParquetWriter.java | 4 +-
.../org/apache/orc/impl/TestPhysicalFsWriter.java | 2 +-
.../src/java/org/apache/orc/tools/Driver.java | 26 +++---
.../src/java/org/apache/orc/tools/FileDump.java | 98 +++++++++++-----------
.../java/org/apache/orc/tools/JsonFileDump.java | 6 +-
.../apache/orc/tools/json/JsonSchemaFinder.java | 4 +-
7 files changed, 74 insertions(+), 70 deletions(-)
diff --git a/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetReader.java b/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetReader.java
index 035ee86..1f071a3 100644
--- a/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetReader.java
+++ b/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetReader.java
@@ -26,6 +26,7 @@ import org.apache.orc.TypeDescription;
import org.apache.orc.bench.core.convert.BatchReader;
import org.apache.orc.bench.core.convert.avro.AvroReader;
import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
import java.io.IOException;
@@ -37,7 +38,8 @@ public class ParquetReader implements BatchReader {
public ParquetReader(Path path,
TypeDescription schema,
Configuration conf) throws IOException {
- reader = AvroParquetReader.<GenericData.Record>builder(path)
+ HadoopInputFile inputFile = HadoopInputFile.fromPath(path, conf);
+ reader = AvroParquetReader.<GenericData.Record>builder(inputFile)
.withCompatibility(true).build();
converters = AvroReader.buildConverters(schema);
}
diff --git a/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetWriter.java b/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetWriter.java
index a1d3648..82edf15 100644
--- a/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetWriter.java
+++ b/java/bench/core/src/java/org/apache/orc/bench/core/convert/parquet/ParquetWriter.java
@@ -30,6 +30,7 @@ import org.apache.orc.bench.core.convert.avro.AvroSchemaUtils;
import org.apache.orc.bench.core.convert.avro.AvroWriter;
import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.util.HadoopOutputFile;
import java.io.IOException;
@@ -60,8 +61,9 @@ public class ParquetWriter implements BatchWriter {
CompressionKind compression
) throws IOException {
Schema avroSchema = AvroSchemaUtils.createAvroSchema(schema);
+ HadoopOutputFile outputFile = HadoopOutputFile.fromPath(path, conf);
writer = AvroParquetWriter
- .<GenericData.Record>builder(path)
+ .<GenericData.Record>builder(outputFile)
.withSchema(avroSchema)
.withConf(conf)
.withCompressionCodec(getParquetCompression(compression))
diff --git a/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java b/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java
index 9ad856c..02dcb0f 100644
--- a/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java
+++ b/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java
@@ -92,7 +92,7 @@ public class TestPhysicalFsWriter {
Progressable progress) throws IOException {
List<byte[]> contents = new ArrayList<>();
fileContents.put(f, contents);
- return new FSDataOutputStream(new MemoryOutputStream(contents));
+ return new FSDataOutputStream(new MemoryOutputStream(contents), null);
}
@Override
diff --git a/java/tools/src/java/org/apache/orc/tools/Driver.java b/java/tools/src/java/org/apache/orc/tools/Driver.java
index 4be6d8c..8e1850b 100644
--- a/java/tools/src/java/org/apache/orc/tools/Driver.java
+++ b/java/tools/src/java/org/apache/orc/tools/Driver.java
@@ -19,8 +19,8 @@
package org.apache.orc.tools;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
@@ -39,17 +39,17 @@ public class Driver {
static Options createOptions() {
Options result = new Options();
- result.addOption(OptionBuilder
- .withLongOpt("help")
- .withDescription("Print help message")
- .create('h'));
+ result.addOption(Option.builder("h")
+ .longOpt("help")
+ .desc("Print help message")
+ .build());
- result.addOption(OptionBuilder
- .withLongOpt("define")
- .withDescription("Set a configuration property")
- .hasArgs(2)
- .withValueSeparator()
- .create('D'));
+ result.addOption(Option.builder("D")
+ .longOpt("define")
+ .desc("Set a configuration property")
+ .numberOfArgs(2)
+ .valueSeparator()
+ .build());
return result;
}
@@ -59,7 +59,7 @@ public class Driver {
final String[] commandArgs;
DriverOptions(String[] args) throws ParseException {
- genericOptions = new GnuParser().parse(createOptions(), args, true);
+ genericOptions = new DefaultParser().parse(createOptions(), args, true);
String[] unprocessed = genericOptions.getArgs();
if (unprocessed.length == 0) {
command = null;
diff --git a/java/tools/src/java/org/apache/orc/tools/FileDump.java b/java/tools/src/java/org/apache/orc/tools/FileDump.java
index 771e60e..1c40585 100644
--- a/java/tools/src/java/org/apache/orc/tools/FileDump.java
+++ b/java/tools/src/java/org/apache/orc/tools/FileDump.java
@@ -26,9 +26,9 @@ import java.util.Collection;
import java.util.List;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -80,7 +80,7 @@ public final class FileDump {
public static void main(Configuration conf, String[] args) throws Exception {
List<Integer> rowIndexCols = new ArrayList<Integer>(0);
Options opts = createOptions();
- CommandLine cli = new GnuParser().parse(opts, args);
+ CommandLine cli = new DefaultParser().parse(opts, args);
if (cli.hasOption('h')) {
HelpFormatter formatter = new HelpFormatter();
@@ -250,10 +250,10 @@ public final class FileDump {
List<String> filesInPath = new ArrayList<>();
FileSystem fs = path.getFileSystem(conf);
FileStatus fileStatus = fs.getFileStatus(path);
- if (fileStatus.isDir()) {
+ if (fileStatus.isDirectory()) {
FileStatus[] fileStatuses = fs.listStatus(path, HIDDEN_AND_SIDE_FILE_FILTER);
for (FileStatus fileInPath : fileStatuses) {
- if (fileInPath.isDir()) {
+ if (fileInPath.isDirectory()) {
filesInPath.addAll(getAllFilesInPath(fileInPath.getPath(), conf));
} else {
filesInPath.add(fileInPath.getPath().toString());
@@ -756,55 +756,55 @@ public final class FileDump {
Options result = new Options();
// add -d and --data to print the rows
- result.addOption(OptionBuilder
- .withLongOpt("data")
- .withDescription("Should the data be printed")
- .create('d'));
+ result.addOption(Option.builder("d")
+ .longOpt("data")
+ .desc("Should the data be printed")
+ .build());
// to avoid breaking unit tests (when run in different time zones) for file dump, printing
// of timezone is made optional
- result.addOption(OptionBuilder
- .withLongOpt("timezone")
- .withDescription("Print writer's time zone")
- .create('t'));
-
- result.addOption(OptionBuilder
- .withLongOpt("help")
- .withDescription("print help message")
- .create('h'));
-
- result.addOption(OptionBuilder
- .withLongOpt("rowindex")
- .withArgName("comma separated list of column ids for which row index should be printed")
- .withDescription("Dump stats for column number(s)")
+ result.addOption(Option.builder("t")
+ .longOpt("timezone")
+ .desc("Print writer's time zone")
+ .build());
+
+ result.addOption(Option.builder("h")
+ .longOpt("help")
+ .desc("Print help message")
+ .build());
+
+ result.addOption(Option.builder("r")
+ .longOpt("rowindex")
+ .argName("comma separated list of column ids for which row index should be printed")
+ .desc("Dump stats for column number(s)")
.hasArg()
- .create('r'));
-
- result.addOption(OptionBuilder
- .withLongOpt("json")
- .withDescription("Print metadata in JSON format")
- .create('j'));
-
- result.addOption(OptionBuilder
- .withLongOpt("pretty")
- .withDescription("Pretty print json metadata output")
- .create('p'));
-
- result.addOption(OptionBuilder
- .withLongOpt("recover")
- .withDescription("recover corrupted orc files generated by streaming")
- .create());
-
- result.addOption(OptionBuilder
- .withLongOpt("skip-dump")
- .withDescription("used along with --recover to directly recover files without dumping")
- .create());
-
- result.addOption(OptionBuilder
- .withLongOpt("backup-path")
- .withDescription("specify a backup path to store the corrupted files (default: /tmp)")
+ .build());
+
+ result.addOption(Option.builder("j")
+ .longOpt("json")
+ .desc("Print metadata in JSON format")
+ .build());
+
+ result.addOption(Option.builder("p")
+ .longOpt("pretty")
+ .desc("Pretty print json metadata output")
+ .build());
+
+ result.addOption(Option.builder()
+ .longOpt("recover")
+ .desc("recover corrupted orc files generated by streaming")
+ .build());
+
+ result.addOption(Option.builder()
+ .longOpt("skip-dump")
+ .desc("used along with --recover to directly recover files without dumping")
+ .build());
+
+ result.addOption(Option.builder()
+ .longOpt("backup-path")
+ .desc("specify a backup path to store the corrupted files (default: /tmp)")
.hasArg()
- .create());
+ .build());
return result;
}
diff --git a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
index a2664f8..ddc1806 100644
--- a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
+++ b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
@@ -362,9 +362,9 @@ public class JsonFileDump {
writer.key("totalLength").value(((StringColumnStatistics) cs).getSum());
writer.key("type").value(OrcProto.Type.Kind.STRING);
} else if (cs instanceof DateColumnStatistics) {
- if (((DateColumnStatistics) cs).getMaximum() != null) {
- writer.key("min").value(((DateColumnStatistics) cs).getMinimum());
- writer.key("max").value(((DateColumnStatistics) cs).getMaximum());
+ if (((DateColumnStatistics) cs).getMaximumLocalDate() != null) {
+ writer.key("min").value(((DateColumnStatistics) cs).getMinimumLocalDate());
+ writer.key("max").value(((DateColumnStatistics) cs).getMaximumLocalDate());
}
writer.key("type").value(OrcProto.Type.Kind.DATE);
} else if (cs instanceof TimestampColumnStatistics) {
diff --git a/java/tools/src/java/org/apache/orc/tools/json/JsonSchemaFinder.java b/java/tools/src/java/org/apache/orc/tools/json/JsonSchemaFinder.java
index 3af9815..1d911a0 100644
--- a/java/tools/src/java/org/apache/orc/tools/json/JsonSchemaFinder.java
+++ b/java/tools/src/java/org/apache/orc/tools/json/JsonSchemaFinder.java
@@ -23,7 +23,7 @@ import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonStreamParser;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
@@ -355,7 +355,7 @@ public class JsonSchemaFinder {
.desc("Print types as Hive table declaration").build());
options.addOption(Option.builder("p").longOpt("pretty")
.desc("Pretty print the schema").build());
- CommandLine cli = new GnuParser().parse(options, args);
+ CommandLine cli = new DefaultParser().parse(options, args);
if (cli.hasOption('h') || cli.getArgs().length == 0) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("json-schema", options);