You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/06/21 23:24:06 UTC
[19/50] [abbrv] hadoop git commit: HADOOP-14424. Add CRC32C
performance test. Contributed by LiXin Ge.
HADOOP-14424. Add CRC32C performance test. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75043d34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75043d34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75043d34
Branch: refs/heads/HDFS-10467
Commit: 75043d342935828cfe8bdabfab8784c83f797887
Parents: 7582ded
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Mon Jun 19 09:16:45 2017 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Mon Jun 19 09:16:45 2017 +0900
----------------------------------------------------------------------
.../hadoop/util/Crc32PerformanceTest.java | 253 +++++++++++++------
1 file changed, 170 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75043d34/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
index d8963df..34dfc3a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.ChecksumException;
import org.apache.log4j.Level;
/**
- * Performance tests to compare performance of Crc32 implementations
+ * Performance tests to compare performance of Crc32|Crc32C implementations
* This can be run from the command line with:
*
* java -cp path/to/test/classes:path/to/common/classes \
@@ -43,17 +43,21 @@ import org.apache.log4j.Level;
*
* hadoop org.apache.hadoop.util.Crc32PerformanceTest
*
+ * If any argument is provided, this test will run with non-directly buffer.
+ *
* The output is in JIRA table format.
*/
public class Crc32PerformanceTest {
static final int MB = 1024 * 1024;
- static interface Crc32 {
+ interface Crc32 {
- public void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
+ void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
String filename, long basePos) throws ChecksumException;
- static final class Native implements Crc32 {
+ DataChecksum.Type crcType();
+
+ final class Native implements Crc32 {
@Override
public void verifyChunked(ByteBuffer data, int bytesPerSum,
ByteBuffer sums, String fileName, long basePos)
@@ -61,40 +65,91 @@ public class Crc32PerformanceTest {
NativeCrc32.verifyChunkedSums(bytesPerSum, DataChecksum.Type.CRC32.id,
sums, data, fileName, basePos);
}
+
+ @Override
+ public DataChecksum.Type crcType() {
+ return DataChecksum.Type.CRC32;
+ }
}
+ final class NativeC implements Crc32 {
+ @Override
+ public void verifyChunked(ByteBuffer data, int bytesPerSum,
+ ByteBuffer sums, String fileName, long basePos)
+ throws ChecksumException {
+
+ if (data.isDirect()) {
+ NativeCrc32.verifyChunkedSums(bytesPerSum,
+ DataChecksum.Type.CRC32C.id, sums, data, fileName, basePos);
+ } else {
+ final int dataOffset = data.arrayOffset() + data.position();
+ final int crcsOffset = sums.arrayOffset() + sums.position();
+ NativeCrc32.verifyChunkedSumsByteArray(bytesPerSum,
+ DataChecksum.Type.CRC32C.id, sums.array(), crcsOffset,
+ data.array(), dataOffset, data.remaining(), fileName, basePos);
+ }
+ }
- static abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
+ @Override
+ public DataChecksum.Type crcType() {
+ return DataChecksum.Type.CRC32C;
+ }
+ }
+
+ abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
abstract T newAlgorithm();
@Override
public void verifyChunked(ByteBuffer data, int bytesPerCrc,
- ByteBuffer crcs, String filename, long basePos)
+ ByteBuffer sums, String filename, long basePos)
throws ChecksumException {
final Checksum algorithm = newAlgorithm();
- if (data.hasArray() && crcs.hasArray()) {
- DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
- data.array(), data.position(), data.remaining(), bytesPerCrc,
- crcs.array(), crcs.position(), filename, basePos);
+ final DataChecksum.Type type = crcType();
+ if (data.hasArray() && sums.hasArray()) {
+ DataChecksum.verifyChunked(type, algorithm, data.array(),
+ data.position(), data.remaining(), bytesPerCrc, sums.array(),
+ sums.position(), filename, basePos);
} else {
- DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
- data, bytesPerCrc, crcs, filename, basePos);
+ DataChecksum.verifyChunked(type, algorithm, data, bytesPerCrc,
+ sums, filename, basePos);
}
}
}
- static final class Zip extends AbstractCrc32<CRC32> {
+ final class Zip extends AbstractCrc32<CRC32> {
@Override
public CRC32 newAlgorithm() {
return new CRC32();
}
+
+ @Override
+ public DataChecksum.Type crcType() {
+ return DataChecksum.Type.CRC32;
+ }
}
- static final class PureJava extends AbstractCrc32<PureJavaCrc32> {
+ final class PureJava extends AbstractCrc32<PureJavaCrc32> {
@Override
public PureJavaCrc32 newAlgorithm() {
return new PureJavaCrc32();
}
+
+ @Override
+ public DataChecksum.Type crcType() {
+ return DataChecksum.Type.CRC32;
+ }
+ }
+
+ final class PureJavaC extends AbstractCrc32<PureJavaCrc32C> {
+ @Override
+ public PureJavaCrc32C newAlgorithm() {
+ return new PureJavaCrc32C();
+ }
+
+ @Override
+ public DataChecksum.Type crcType() {
+ return DataChecksum.Type.CRC32C;
+ }
}
}
@@ -114,9 +169,13 @@ public class Crc32PerformanceTest {
crcs.add(Crc32.Zip.class);
crcs.add(Crc32.PureJava.class);
+ crcs.add(Crc32.PureJavaC.class);
- if (direct && NativeCrc32.isAvailable()) {
- crcs.add(Crc32.Native.class);
+ if (NativeCrc32.isAvailable()) {
+ if (direct) {
+ crcs.add(Crc32.Native.class);
+ }
+ crcs.add(Crc32.NativeC.class);
((Log4JLogger)LogFactory.getLog(NativeCodeLoader.class))
.getLogger().setLevel(Level.ALL);
}
@@ -131,13 +190,18 @@ public class Crc32PerformanceTest {
out.printf("Elapsed %.1fs\n", secondsElapsed(startTime));
}
- public static void main(String args[]) throws Exception {
- new Crc32PerformanceTest(64, 5, true).run();
+ public static void main(String[] args) throws Exception {
+ boolean isdirect = true;
+
+ if (args.length > 0) {
+ isdirect = false;
+ }
+ new Crc32PerformanceTest(64, 5, isdirect).run();
}
- private static void printCell(String s, int width, PrintStream out) {
+ private static void printCell(String s, int width, PrintStream outCrc) {
final int w = s.length() > width? s.length(): width;
- out.printf(" %" + w + "s |", s);
+ outCrc.printf(" %" + w + "s |", s);
}
private ByteBuffer allocateByteBuffer(int length) {
@@ -155,16 +219,29 @@ public class Crc32PerformanceTest {
return dataBufs;
}
- private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc) {
+ private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc,
+ DataChecksum.Type type) {
+ final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
+ final ByteBuffer crcBufs = allocateByteBuffer(size);
+ final DataChecksum checksum = DataChecksum.newDataChecksum(
+ type, bytePerCrc);
+ checksum.calculateChunkedSums(dataBufs, crcBufs);
+ return crcBufs;
+ }
+
+ private ByteBuffer computeCrc(Class<? extends Crc32> clazz,
+ ByteBuffer dataBufs, int bytePerCrc) throws Exception {
+ final Constructor<? extends Crc32> ctor = clazz.getConstructor();
+ final Crc32 crc = ctor.newInstance();
final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
final ByteBuffer crcBufs = allocateByteBuffer(size);
final DataChecksum checksum = DataChecksum.newDataChecksum(
- DataChecksum.Type.CRC32, bytePerCrc);
+ crc.crcType(), bytePerCrc);
checksum.calculateChunkedSums(dataBufs, crcBufs);
return crcBufs;
}
- private void doBench(final List<Class<? extends Crc32>> crcs)
+ private void doBench(final List<Class<? extends Crc32>> crcTargets)
throws Exception {
final ByteBuffer[] dataBufs = new ByteBuffer[16];
for(int i = 0; i < dataBufs.length; i++) {
@@ -176,9 +253,9 @@ public class Crc32PerformanceTest {
out.printf(" (bpc: byte-per-crc in MB/sec; #T: #Theads)\n");
// Warm up implementations to get jit going.
- final ByteBuffer[] crc32 = {computeCrc(dataBufs[0], 32)};
- final ByteBuffer[] crc512 = {computeCrc(dataBufs[0], 512)};
- for (Class<? extends Crc32> c : crcs) {
+ for (Class<? extends Crc32> c : crcTargets) {
+ final ByteBuffer[] crc32 = {computeCrc(c, dataBufs[0], 32)};
+ final ByteBuffer[] crc512 = {computeCrc(c, dataBufs[0], 512)};
doBench(c, 1, dataBufs, crc32, 32);
doBench(c, 1, dataBufs, crc512, 512);
}
@@ -189,57 +266,69 @@ public class Crc32PerformanceTest {
}
}
- private void doBench(final List<Class<? extends Crc32>> crcs,
- final ByteBuffer[] dataBufs, final int bytePerCrc, final PrintStream out)
+ private void doBench(final List<Class<? extends Crc32>> crcTargets,
+ final ByteBuffer[] dataBufs, final int bytePerCrc,
+ final PrintStream outCrc)
throws Exception {
final ByteBuffer[] crcBufs = new ByteBuffer[dataBufs.length];
- for(int i = 0; i < crcBufs.length; i++) {
- crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc);
+ final ByteBuffer[] crcBufsC = new ByteBuffer[dataBufs.length];
+ for(int i = 0; i < dataBufs.length; i++) {
+ crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc,
+ DataChecksum.Type.CRC32);
+ crcBufsC[i] = computeCrc(dataBufs[i], bytePerCrc,
+ DataChecksum.Type.CRC32C);
}
final String numBytesStr = " bpc ";
final String numThreadsStr = "#T";
final String diffStr = "% diff";
- out.print('|');
- printCell(numBytesStr, 0, out);
- printCell(numThreadsStr, 0, out);
- for (int i = 0; i < crcs.size(); i++) {
- final Class<? extends Crc32> c = crcs.get(i);
- out.print('|');
- printCell(c.getSimpleName(), 8, out);
- for(int j = 0; j < i; j++) {
- printCell(diffStr, diffStr.length(), out);
+ outCrc.print('|');
+ printCell(numBytesStr, 0, outCrc);
+ printCell(numThreadsStr, 0, outCrc);
+ for (int i = 0; i < crcTargets.size(); i++) {
+ final Class<? extends Crc32> c = crcTargets.get(i);
+ outCrc.print('|');
+ printCell(c.getSimpleName(), 8, outCrc);
+ if (i > 0) {
+ printCell(diffStr, diffStr.length(), outCrc);
}
}
- out.printf("\n");
+ outCrc.printf("\n");
for(int numThreads = 1; numThreads <= dataBufs.length; numThreads <<= 1) {
- out.printf("|");
- printCell(String.valueOf(bytePerCrc), numBytesStr.length(), out);
- printCell(String.valueOf(numThreads), numThreadsStr.length(), out);
+ outCrc.printf("|");
+ printCell(String.valueOf(bytePerCrc), numBytesStr.length(), outCrc);
+ printCell(String.valueOf(numThreads), numThreadsStr.length(), outCrc);
final List<BenchResult> previous = new ArrayList<BenchResult>();
- for(Class<? extends Crc32> c : crcs) {
+ for(Class<? extends Crc32> c : crcTargets) {
System.gc();
- final BenchResult result = doBench(c, numThreads, dataBufs, crcBufs,
- bytePerCrc);
+ final BenchResult result;
+ final Constructor<? extends Crc32> ctor = c.getConstructor();
+ final Crc32 crc = ctor.newInstance();
+ if (crc.crcType() == DataChecksum.Type.CRC32) {
+ result = doBench(c, numThreads, dataBufs, crcBufs, bytePerCrc);
+ } else {
+ result = doBench(c, numThreads, dataBufs, crcBufsC, bytePerCrc);
+ }
printCell(String.format("%9.1f", result.mbps),
- c.getSimpleName().length() + 1, out);
+ c.getSimpleName().length() + 1, outCrc);
- //compare result with previous
- for(BenchResult p : previous) {
+ //compare result with the last previous.
+ final int size = previous.size();
+ if (size > 0) {
+ BenchResult p = previous.get(size - 1);
final double diff = (result.mbps - p.mbps) / p.mbps * 100;
- printCell(String.format("%5.1f%%", diff), diffStr.length(), out);
+ printCell(String.format("%5.1f%%", diff), diffStr.length(), outCrc);
}
previous.add(result);
}
- out.printf("\n");
+ outCrc.printf("\n");
}
}
-
private BenchResult doBench(Class<? extends Crc32> clazz,
final int numThreads, final ByteBuffer[] dataBufs,
final ByteBuffer[] crcBufs, final int bytePerCrc)
@@ -248,36 +337,34 @@ public class Crc32PerformanceTest {
final Thread[] threads = new Thread[numThreads];
final BenchResult[] results = new BenchResult[threads.length];
- {
- final Constructor<? extends Crc32> ctor = clazz.getConstructor();
-
- for(int i = 0; i < threads.length; i++) {
- final Crc32 crc = ctor.newInstance();
- final long byteProcessed = dataBufs[i].remaining() * trials;
- final int index = i;
- threads[i] = new Thread() {
- @Override
- public void run() {
- final long startTime = System.nanoTime();
- for (int i = 0; i < trials; i++) {
- dataBufs[index].mark();
- crcBufs[index].mark();
- try {
- crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index],
- crc.getClass().getSimpleName(), dataBufs[index].position());
- } catch (Throwable t) {
- results[index] = new BenchResult(t);
- return;
- } finally {
- dataBufs[index].reset();
- crcBufs[index].reset();
- }
+ final Constructor<? extends Crc32> ctor = clazz.getConstructor();
+
+ for(int i = 0; i < threads.length; i++) {
+ final Crc32 crc = ctor.newInstance();
+ final long byteProcessed = dataBufs[i].remaining() * trials;
+ final int index = i;
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ final long startTime = System.nanoTime();
+ for (int i = 0; i < trials; i++) {
+ dataBufs[index].mark();
+ crcBufs[index].mark();
+ try {
+ crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index],
+ crc.getClass().getSimpleName(), dataBufs[index].position());
+ } catch (Throwable t) {
+ results[index] = new BenchResult(t);
+ return;
+ } finally {
+ dataBufs[index].reset();
+ crcBufs[index].reset();
}
- final double secsElapsed = secondsElapsed(startTime);
- results[index] = new BenchResult(byteProcessed/secsElapsed/MB);
}
- };
- }
+ final double secsElapsed = secondsElapsed(startTime);
+ results[index] = new BenchResult(byteProcessed/secsElapsed/MB);
+ }
+ };
}
for(Thread t : threads) {
@@ -295,7 +382,7 @@ public class Crc32PerformanceTest {
}
private static class BenchResult {
- /** Speed (MB per second) */
+ /** Speed (MB per second). */
final double mbps;
final Throwable thrown;
@@ -321,7 +408,7 @@ public class Crc32PerformanceTest {
return (System.nanoTime() - startTime) / 1000000000.0d;
}
- static void printSystemProperties(PrintStream out) {
+ static void printSystemProperties(PrintStream outCrc) {
final String[] names = {
"java.version",
"java.runtime.name",
@@ -344,7 +431,7 @@ public class Crc32PerformanceTest {
final Properties p = System.getProperties();
for(String n : names) {
- out.printf("%" + max + "s = %s\n", n, p.getProperty(n));
+ outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n));
}
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org