You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by pg...@apache.org on 2021/03/23 13:30:28 UTC
[orc] branch master updated: ORC-770: Support ZSTD in Avro data
benchmark (#665)
This is an automated email from the ASF dual-hosted git repository.
pgaref pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/master by this push:
new 60b1d73 ORC-770: Support ZSTD in Avro data benchmark (#665)
60b1d73 is described below
commit 60b1d73d40f42f119f4f2830f69d6dee416328a2
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue Mar 23 06:30:21 2021 -0700
ORC-770: Support ZSTD in Avro data benchmark (#665)
---
java/bench/core/pom.xml | 1 -
.../org/apache/orc/bench/core/convert/avro/AvroWriter.java | 3 +++
java/bench/hive/pom.xml | 1 -
java/bench/pom.xml | 11 ++---------
4 files changed, 5 insertions(+), 11 deletions(-)
diff --git a/java/bench/core/pom.xml b/java/bench/core/pom.xml
index 47dbf76..d5fb952 100644
--- a/java/bench/core/pom.xml
+++ b/java/bench/core/pom.xml
@@ -59,7 +59,6 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
- <classifier>hadoop2</classifier>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/java/bench/core/src/java/org/apache/orc/bench/core/convert/avro/AvroWriter.java b/java/bench/core/src/java/org/apache/orc/bench/core/convert/avro/AvroWriter.java
index 13e148e..c509652 100644
--- a/java/bench/core/src/java/org/apache/orc/bench/core/convert/avro/AvroWriter.java
+++ b/java/bench/core/src/java/org/apache/orc/bench/core/convert/avro/AvroWriter.java
@@ -333,6 +333,9 @@ public class AvroWriter implements BatchWriter {
case SNAPPY:
writer.setCodec(CodecFactory.snappyCodec());
break;
+ case ZSTD:
+ writer.setCodec(CodecFactory.zstandardCodec(CodecFactory.DEFAULT_ZSTANDARD_LEVEL));
+ break;
default:
throw new IllegalArgumentException("Compression unsupported " + compression);
}
diff --git a/java/bench/hive/pom.xml b/java/bench/hive/pom.xml
index e8ed1bf..8733273 100644
--- a/java/bench/hive/pom.xml
+++ b/java/bench/hive/pom.xml
@@ -52,7 +52,6 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
- <classifier>hadoop2</classifier>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/java/bench/pom.xml b/java/bench/pom.xml
index d1803c5..4a075f1 100644
--- a/java/bench/pom.xml
+++ b/java/bench/pom.xml
@@ -36,7 +36,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.useIncrementalCompilation>false</maven.compiler.useIncrementalCompilation>
- <avro.version>1.8.2</avro.version>
+ <avro.version>1.10.2</avro.version>
<hive.version>3.1.2</hive.version>
<jmh.version>1.20</jmh.version>
<orc.version>${project.version}</orc.version>
@@ -56,7 +56,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>2.8.4</version>
+ <version>2.12.2</version>
</dependency>
<dependency>
<groupId>com.google.auto.service</groupId>
@@ -103,14 +103,7 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
- <classifier>hadoop2</classifier>
<version>${avro.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>