You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by dk...@apache.org on 2018/11/16 20:49:51 UTC
[avro] branch master updated: Make snnappy/xz/zstd-jni/paranamer
optional/provided in avro jar.
This is an automated email from the ASF dual-hosted git repository.
dkulp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 2109852 Make snnappy/xz/zstd-jni/paranamer optional/provided in avro jar.
2109852 is described below
commit 21098529e659743e551f2c35652d1a93c1528082
Author: Daniel Kulp <dk...@apache.org>
AuthorDate: Fri Nov 16 15:48:57 2018 -0500
Make snnappy/xz/zstd-jni/paranamer optional/provided in avro jar.
---
lang/java/avro/pom.xml | 25 +++++++++++------
.../java/org/apache/avro/file/CodecFactory.java | 17 ++++++++----
.../java/org/apache/avro/file/SnappyCodec.java | 5 ++++
.../main/java/org/apache/avro/file/XZCodec.java | 1 +
.../java/org/apache/avro/reflect/ReflectData.java | 32 ++++++++++++++++++----
lang/java/mapred/pom.xml | 6 ++++
lang/java/pom.xml | 3 +-
lang/java/tools/pom.xml | 14 ++++++++++
8 files changed, 82 insertions(+), 21 deletions(-)
diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml
index c9104fb..ffa4b69 100644
--- a/lang/java/avro/pom.xml
+++ b/lang/java/avro/pom.xml
@@ -37,8 +37,8 @@
<properties>
<osgi.import>
!org.apache.avro*,
- com.thoughtworks.paranamer,
com.fasterxml.jackson*,
+ com.thoughtworks.paranamer;resolution:=optional,
org.xerial.snappy;resolution:=optional,
sun.misc;resolution:=optional,
*
@@ -151,19 +151,30 @@
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
- </dependency>
- <dependency>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
+ <optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
<dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
<version>${tukaani.version}</version>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
</dependency>
<dependency>
<groupId>joda-time</groupId>
@@ -175,10 +186,6 @@
<artifactId>hamcrest-all</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.github.luben</groupId>
- <artifactId>zstd-jni</artifactId>
- </dependency>
</dependencies>
</project>
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java b/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
index 238e8a4..1910894 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
@@ -22,7 +22,6 @@ import java.util.Map;
import java.util.zip.Deflater;
import org.apache.avro.AvroRuntimeException;
-import org.tukaani.xz.LZMA2Options;
/** Encapsulates the ability to specify and configure a compression codec.
*
@@ -57,7 +56,12 @@ public abstract class CodecFactory {
/** Snappy codec.*/
public static CodecFactory snappyCodec() {
- return new SnappyCodec.Option();
+ try {
+ return new SnappyCodec.Option();
+ } catch (Throwable t) {
+ //snappy not available
+ return null;
+ }
}
/** bzip2 codec.*/
@@ -80,15 +84,15 @@ public abstract class CodecFactory {
new HashMap<>();
public static final int DEFAULT_DEFLATE_LEVEL = Deflater.DEFAULT_COMPRESSION;
- public static final int DEFAULT_XZ_LEVEL = LZMA2Options.PRESET_DEFAULT;
+ public static final int DEFAULT_XZ_LEVEL = XZCodec.DEFAULT_COMPRESSION;
static {
addCodec(DataFileConstants.NULL_CODEC, nullCodec());
addCodec(DataFileConstants.DEFLATE_CODEC, deflateCodec(DEFAULT_DEFLATE_LEVEL));
- addCodec(DataFileConstants.SNAPPY_CODEC, snappyCodec());
addCodec(DataFileConstants.BZIP2_CODEC, bzip2Codec());
addCodec(DataFileConstants.XZ_CODEC, xzCodec(DEFAULT_XZ_LEVEL));
addCodec(DataFileConstants.ZSTANDARD_CODEC, zstandardCodec());
+ addCodec(DataFileConstants.SNAPPY_CODEC, snappyCodec());
}
/** Maps a codec name into a CodecFactory.
@@ -116,7 +120,10 @@ public abstract class CodecFactory {
/** Adds a new codec implementation. If name already had
* a codec associated with it, returns the previous codec. */
public static CodecFactory addCodec(String name, CodecFactory c) {
- return REGISTERED.put(name, c);
+ if (c != null) {
+ return REGISTERED.put(name, c);
+ }
+ return null;
}
@Override
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java b/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java
index 04f7218..4ffdce7 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java
@@ -28,6 +28,11 @@ public class SnappyCodec extends Codec {
private CRC32 crc32 = new CRC32();
static class Option extends CodecFactory {
+ static {
+ //if snappy isn't available, this will throw an exception which we
+ //can catch so we can avoid registering this codec
+ Snappy.getNativeLibraryVersion();
+ }
@Override
protected Codec createInstance() {
return new SnappyCodec();
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java b/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java
index 92a742a..06a3bc6 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java
@@ -30,6 +30,7 @@ import org.apache.commons.compress.utils.IOUtils;
/** * Implements xz compression and decompression. */
public class XZCodec extends Codec {
+ public final static int DEFAULT_COMPRESSION = 6;
static class Option extends CodecFactory {
private int compressionLevel;
diff --git a/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java b/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
index d26d1f9..12d3e15 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
import org.apache.avro.AvroRemoteException;
import org.apache.avro.AvroRuntimeException;
@@ -60,9 +61,6 @@ import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.SchemaNormalization;
-import com.thoughtworks.paranamer.CachingParanamer;
-import com.thoughtworks.paranamer.Paranamer;
-
/** Utilities to use existing Java classes and interfaces via reflection. */
public class ReflectData extends SpecificData {
@Override
@@ -790,12 +788,36 @@ public class ReflectData extends SpecificData {
return protocol;
}
- private final Paranamer paranamer = new CachingParanamer();
+ private Function<Method, String[]> paranamer;
+ private synchronized Function<Method, String[]> getParanamer() {
+ if (paranamer == null) {
+ try {
+ final com.thoughtworks.paranamer.CachingParanamer p = new com.thoughtworks.paranamer.CachingParanamer();
+ paranamer = new Function<Method, String[]>() {
+ public String[] apply(Method t) {
+ return p.lookupParameterNames(t);
+ }
+ };
+ } catch (Throwable t) {
+ paranamer = new Function<Method, String[]>() {
+ public String[] apply(Method t) {
+ return new String[0];
+ }
+ };
+ }
+ }
+ return paranamer;
+ }
+
+ private String[] getParameterNames(Method m) {
+ return getParanamer().apply(m);
+ }
+
private Message getMessage(Method method, Protocol protocol,
Map<String,Schema> names) {
List<Schema.Field> fields = new ArrayList<>();
- String[] paramNames = paranamer.lookupParameterNames(method);
+ String[] paramNames = getParameterNames(method);
Type[] paramTypes = method.getGenericParameterTypes();
Annotation[][] annotations = method.getParameterAnnotations();
for (int i = 0; i < paramTypes.length; i++) {
diff --git a/lang/java/mapred/pom.xml b/lang/java/mapred/pom.xml
index 368226a..4309b67 100644
--- a/lang/java/mapred/pom.xml
+++ b/lang/java/mapred/pom.xml
@@ -151,6 +151,12 @@
So we should not include it here -->
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</project>
diff --git a/lang/java/pom.xml b/lang/java/pom.xml
index e26b0ed..6a426fe 100644
--- a/lang/java/pom.xml
+++ b/lang/java/pom.xml
@@ -550,8 +550,7 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>${slf4j.version}</version>
- <scope>runtime</scope>
- <optional>true</optional>
+ <scope>test</scope>
</dependency>
</dependencies>
diff --git a/lang/java/tools/pom.xml b/lang/java/tools/pom.xml
index 2137e08..17e0195 100644
--- a/lang/java/tools/pom.xml
+++ b/lang/java/tools/pom.xml
@@ -200,6 +200,20 @@
<artifactId>avro-mapred</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.tukaani</groupId>
+ <artifactId>xz</artifactId>
+ <version>${tukaani.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+
<!--For testing TetherTool we need the mapred test jar
because that contains the word count example.-->