You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/01/11 10:49:15 UTC
hadoop git commit: HADOOP-15033. Use java.util.zip.CRC32C for Java 9
and above Contributed by Dmitry Chuyko,
Repository: hadoop
Updated Branches:
refs/heads/trunk fbbbf59c8 -> 2e0a451a8
HADOOP-15033. Use java.util.zip.CRC32C for Java 9 and above
Contributed by Dmitry Chuyko,
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e0a451a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e0a451a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e0a451a
Branch: refs/heads/trunk
Commit: 2e0a451a8dd64ccd7199b2b435e1b6c704a9d3ae
Parents: fbbbf59
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jan 11 10:47:50 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jan 11 10:47:50 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/util/DataChecksum.java | 60 +++++++++++++++++++-
.../main/java/org/apache/hadoop/util/Shell.java | 15 +++++
.../hadoop/util/Crc32PerformanceTest.java | 17 +++++-
.../java/org/apache/hadoop/util/TestShell.java | 5 ++
hadoop-project/pom.xml | 1 +
5 files changed, 96 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e0a451a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index 03946af..43e377f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -28,6 +28,12 @@ import java.util.zip.Checksum;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.ChecksumException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
/**
* This class provides interface and utilities for processing checksums for
@@ -43,6 +49,9 @@ public class DataChecksum implements Checksum {
public static final int CHECKSUM_CRC32C = 2;
public static final int CHECKSUM_DEFAULT = 3;
public static final int CHECKSUM_MIXED = 4;
+
+ private static final Logger LOG = LoggerFactory.getLogger(DataChecksum.class);
+ private static volatile boolean useJava9Crc32C = Shell.isJavaVersionAtLeast(9);
/** The checksum types */
public enum Type {
@@ -78,6 +87,23 @@ public class DataChecksum implements Checksum {
return new CRC32();
}
+
+ /**
+ * The flag is volatile to avoid synchronization here.
+ * Re-entrancy is unlikely except in failure mode (and inexpensive).
+ */
+ static Checksum newCrc32C() {
+ try {
+ return useJava9Crc32C ? Java9Crc32CFactory.createChecksum()
+ : new PureJavaCrc32C();
+ } catch (ExceptionInInitializerError | RuntimeException e) {
+ // should not happen
+ LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e);
+ useJava9Crc32C = false;
+ return new PureJavaCrc32C();
+ }
+ }
+
public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
if ( bytesPerChecksum <= 0 ) {
return null;
@@ -89,7 +115,7 @@ public class DataChecksum implements Checksum {
case CRC32 :
return new DataChecksum(type, newCrc32(), bytesPerChecksum );
case CRC32C:
- return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+ return new DataChecksum(type, newCrc32C(), bytesPerChecksum);
default:
return null;
}
@@ -528,4 +554,36 @@ public class DataChecksum implements Checksum {
@Override
public void update(int b) {}
};
+
+ /**
+ * Holds constructor handle to let it be initialized on demand.
+ */
+ private static class Java9Crc32CFactory {
+ private static final MethodHandle NEW_CRC32C_MH;
+
+ static {
+ MethodHandle newCRC32C = null;
+ try {
+ newCRC32C = MethodHandles.publicLookup()
+ .findConstructor(
+ Class.forName("java.util.zip.CRC32C"),
+ MethodType.methodType(void.class)
+ );
+ } catch (ReflectiveOperationException e) {
+ // Should not reach here.
+ throw new RuntimeException(e);
+ }
+ NEW_CRC32C_MH = newCRC32C;
+ }
+
+ public static Checksum createChecksum() {
+ try {
+ // Should throw nothing
+ return (Checksum) NEW_CRC32C_MH.invoke();
+ } catch (Throwable t) {
+ throw (t instanceof RuntimeException) ? (RuntimeException) t
+ : new RuntimeException(t);
+ }
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e0a451a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index bfb8183..c25cba2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -89,6 +89,21 @@ public abstract class Shell {
return true;
}
+ // "1.8"->8, "9"->9, "10"->10
+ private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt(
+ System.getProperty("java.specification.version").split("\\.")[0]));
+
+ /**
+ * Query to see if major version of Java specification of the system
+ * is equal or greater than the parameter.
+ *
+ * @param version 8, 9, 10 etc.
+ * @return comparison with system property, always true for 8
+ */
+ public static boolean isJavaVersionAtLeast(int version) {
+ return JAVA_SPEC_VER >= version;
+ }
+
/**
* Maximum command line length in Windows
* KB830473 documents this as 8191
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e0a451a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
index ce28f50..f0d6145 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
@@ -129,6 +129,18 @@ public class Crc32PerformanceTest {
}
}
+ final class ZipC extends AbstractCrc32<Checksum> {
+ @Override
+ public Checksum newAlgorithm() {
+ return DataChecksum.newCrc32C();
+ }
+
+ @Override
+ public DataChecksum.Type crcType() {
+ return DataChecksum.Type.CRC32C;
+ }
+ }
+
final class PureJava extends AbstractCrc32<PureJavaCrc32> {
@Override
public PureJavaCrc32 newAlgorithm() {
@@ -169,6 +181,9 @@ public class Crc32PerformanceTest {
this.direct = direct;
crcs.add(Crc32.Zip.class);
+ if (Shell.isJavaVersionAtLeast(9)) {
+ crcs.add(Crc32.ZipC.class);
+ }
crcs.add(Crc32.PureJava.class);
crcs.add(Crc32.PureJavaC.class);
@@ -435,4 +450,4 @@ public class Crc32PerformanceTest {
outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n));
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e0a451a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index 4f30edb..d0ebc2b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -523,4 +523,9 @@ public class TestShell extends Assert {
shexc1.getProcess().waitFor();
shexc2.getProcess().waitFor();
}
+
+ @Test
+ public void testIsJavaVersionAtLeast() {
+ assertTrue(Shell.isJavaVersionAtLeast(8));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e0a451a/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3c49182..a73b6af 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1571,6 +1571,7 @@
<ignore>sun.nio.ch.*</ignore>
<ignore>com.sun.javadoc.*</ignore>
<ignore>com.sun.tools.*</ignore>
+ <ignore>java.lang.invoke.*</ignore>
</ignores>
</configuration>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org