You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2021/02/25 17:56:01 UTC
[ozone] branch master updated: HDDS-4138. Improve crc efficiency by
using Java.util.zip.CRC when available (#1950)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 85e27c3 HDDS-4138. Improve crc efficiency by using Java.util.zip.CRC when available (#1950)
85e27c3 is described below
commit 85e27c3a47ec1a49bf9a284070fa86e8a3138ba6
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Thu Feb 25 17:55:43 2021 +0000
HDDS-4138. Improve crc efficiency by using Java.util.zip.CRC when available (#1950)
---
.../java/org/apache/hadoop/hdds/JavaUtils.java | 44 ++++++++++++++
.../org/apache/hadoop/ozone/common/Checksum.java | 6 +-
...ferImpl.java => ChecksumByteBufferFactory.java} | 71 ++++++++++------------
.../ozone/common/ChecksumByteBufferImpl.java | 32 ----------
.../ozone/common/TestChecksumByteBuffer.java | 4 +-
.../common/TestChecksumImplsComputeSameValues.java | 2 +-
.../ozone/genesis/BenchMarkCRCStreaming.java | 3 +-
7 files changed, 86 insertions(+), 76 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java
new file mode 100644
index 0000000..63c29ba
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds;
+
+/**
+ * Various reusable utility methods related to Java.
+ */
+public final class JavaUtils {
+ // "1.8"->8, "9"->9, "10"->10
+ private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt(
+ System.getProperty("java.specification.version").split("\\.")[0]));
+
+ /**
+ * Query to see if major version of Java specification of the system
+ * is equal or greater than the parameter.
+ *
+ * @param version 8, 9, 10 etc.
+ * @return comparison with system property, always true for 8
+ */
+ public static boolean isJavaVersionAtLeast(int version) {
+ return JAVA_SPEC_VER >= version;
+ }
+
+ /**
+ * Private constructor.
+ */
+ private JavaUtils() {
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
index d86f7b1..db7a31e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
@@ -75,8 +75,10 @@ public class Checksum {
/** The algorithms for {@link ChecksumType}. */
enum Algorithm {
NONE(() -> data -> ByteString.EMPTY),
- CRC32(() -> newChecksumByteBufferFunction(PureJavaCrc32ByteBuffer::new)),
- CRC32C(() -> newChecksumByteBufferFunction(PureJavaCrc32CByteBuffer::new)),
+ CRC32(() ->
+ newChecksumByteBufferFunction(ChecksumByteBufferFactory::crc32Impl)),
+ CRC32C(() ->
+ newChecksumByteBufferFunction(ChecksumByteBufferFactory::crc32CImpl)),
SHA256(() -> newMessageDigestFunction("SHA-256")),
MD5(() -> newMessageDigestFunction("MD5"));
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java
similarity index 63%
copy from hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
copy to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java
index 18651fe..a41af84 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java
@@ -17,13 +17,27 @@
*/
package org.apache.hadoop.ozone.common;
+import org.apache.hadoop.hdds.JavaUtils;
+import org.apache.hadoop.util.PureJavaCrc32C;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
-import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
import java.util.zip.Checksum;
-public class ChecksumByteBufferImpl implements ChecksumByteBuffer {
+/**
+ * Class containing factories for creating various checksum impls.
+ */
+public final class ChecksumByteBufferFactory {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ChecksumByteBufferImpl.class);
+
+ private static volatile boolean useJava9Crc32C
+ = JavaUtils.isJavaVersionAtLeast(9);
public static class Java9Crc32CFactory {
private static final MethodHandle NEW_CRC32C_MH;
@@ -54,45 +68,26 @@ public class ChecksumByteBufferImpl implements ChecksumByteBuffer {
}
};
- private Checksum checksum;
-
- public ChecksumByteBufferImpl(Checksum impl) {
- this.checksum = impl;
+ public static ChecksumByteBuffer crc32Impl() {
+ return new ChecksumByteBufferImpl(new CRC32());
}
- @Override
- // TODO - when we eventually move to a minimum Java version >= 9 this method
- // should be refactored to simply call checksum.update(buffer), as the
- // Checksum interface has been enhanced to allow this since Java 9.
- public void update(ByteBuffer buffer) {
- if (buffer.hasArray()) {
- checksum.update(buffer.array(), buffer.position() + buffer.arrayOffset(),
- buffer.remaining());
- } else {
- byte[] b = new byte[buffer.remaining()];
- buffer.get(b);
- checksum.update(b, 0, b.length);
+ public static ChecksumByteBuffer crc32CImpl() {
+ if (useJava9Crc32C) {
+ try {
+ return new ChecksumByteBufferImpl(Java9Crc32CFactory.createChecksum());
+ } catch (Throwable e) {
+ // should not happen
+ LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e);
+ useJava9Crc32C = false;
+ }
}
+ return new ChecksumByteBufferImpl(new PureJavaCrc32C());
}
- @Override
- public void update(byte[] b, int off, int len) {
- checksum.update(b, off, len);
+ /**
+ * Private Constructor.
+ */
+ private ChecksumByteBufferFactory() {
}
-
- @Override
- public void update(int i) {
- checksum.update(i);
- }
-
- @Override
- public long getValue() {
- return checksum.getValue();
- }
-
- @Override
- public void reset() {
- checksum.reset();
- }
-
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
index 18651fe..db779b6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
@@ -17,43 +17,11 @@
*/
package org.apache.hadoop.ozone.common;
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles;
-import java.lang.invoke.MethodType;
import java.nio.ByteBuffer;
import java.util.zip.Checksum;
public class ChecksumByteBufferImpl implements ChecksumByteBuffer {
- public static class Java9Crc32CFactory {
- private static final MethodHandle NEW_CRC32C_MH;
-
- static {
- MethodHandle newCRC32C = null;
- try {
- newCRC32C = MethodHandles.publicLookup()
- .findConstructor(
- Class.forName("java.util.zip.CRC32C"),
- MethodType.methodType(void.class)
- );
- } catch (ReflectiveOperationException e) {
- // Should not reach here.
- throw new RuntimeException(e);
- }
- NEW_CRC32C_MH = newCRC32C;
- }
-
- public static java.util.zip.Checksum createChecksum() {
- try {
- // Should throw nothing
- return (Checksum) NEW_CRC32C_MH.invoke();
- } catch (Throwable t) {
- throw (t instanceof RuntimeException) ? (RuntimeException) t
- : new RuntimeException(t);
- }
- }
- };
-
private Checksum checksum;
public ChecksumByteBufferImpl(Checksum impl) {
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
index 2f46637..4aac0b4 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
@@ -33,14 +33,14 @@ public class TestChecksumByteBuffer {
@Test
public void testPureJavaCrc32ByteBuffer() {
final Checksum expected = new PureJavaCrc32();
- final ChecksumByteBuffer testee = new PureJavaCrc32ByteBuffer();
+ final ChecksumByteBuffer testee = ChecksumByteBufferFactory.crc32Impl();
new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
@Test
public void testPureJavaCrc32CByteBuffer() {
final Checksum expected = new PureJavaCrc32C();
- final ChecksumByteBuffer testee = new PureJavaCrc32CByteBuffer();
+ final ChecksumByteBuffer testee = ChecksumByteBufferFactory.crc32CImpl();
new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java
index aa4b454..a098a26 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumImplsComputeSameValues.java
@@ -62,7 +62,7 @@ public class TestChecksumImplsComputeSameValues {
impls.add(new ChecksumByteBufferImpl(new PureJavaCrc32C()));
try {
impls.add(new ChecksumByteBufferImpl(
- ChecksumByteBufferImpl.Java9Crc32CFactory.createChecksum()));
+ ChecksumByteBufferFactory.Java9Crc32CFactory.createChecksum()));
} catch (Throwable e) {
// NOOP
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java
index 0d8ef2f..5dd5da8 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkCRCStreaming.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.ozone.common.ChecksumByteBuffer;
+import org.apache.hadoop.ozone.common.ChecksumByteBufferFactory;
import org.apache.hadoop.ozone.common.ChecksumByteBufferImpl;
import org.apache.hadoop.ozone.common.NativeCheckSumCRC32;
import org.apache.hadoop.ozone.common.PureJavaCrc32ByteBuffer;
@@ -114,7 +115,7 @@ public class BenchMarkCRCStreaming {
case "zipCRC32C":
try {
checksum = new ChecksumByteBufferImpl(
- ChecksumByteBufferImpl.Java9Crc32CFactory.createChecksum());
+ ChecksumByteBufferFactory.Java9Crc32CFactory.createChecksum());
} catch (Throwable e) {
throw new RuntimeException("zipCRC32C is not available pre Java 9");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org