You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by mm...@apache.org on 2018/01/02 08:26:39 UTC
[bookkeeper] branch master updated: ISSUE #590 (@bug W-4556980@)
Reduce excessive CPU usage on client side: add crc32c
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 2fbb291 ISSUE #590 (@bug W-4556980@) Reduce excessive CPU usage on client side: add crc32c
2fbb291 is described below
commit 2fbb2916a323edfc097de0de9b2d81e59b37bf6f
Author: Andrey Yegorov <ay...@salesforce.com>
AuthorDate: Tue Jan 2 09:26:31 2018 +0100
ISSUE #590 (@bug W-4556980@) Reduce excessive CPU usage on client side: add crc32c
…2c with SSE processor instruction support, benchmark to make sure it is better than crc32
Descriptions of the changes in this PR:
CRC32C for bookkeeper
Master Issue: #590
Author: Andrey Yegorov <ay...@salesforce.com>
Reviewers: Enrico Olivelli <eo...@gmail.com>, Sijie Guo <si...@apache.org>, Matteo Merli <mm...@apache.org>
This closes #856 from dlg99/feature/crc32c, closes #590
---
bookkeeper-proto/src/main/proto/DataFormats.proto | 1 +
bookkeeper-server/pom.xml | 5 +
.../org/apache/bookkeeper/client/BookKeeper.java | 13 +-
.../bookkeeper/client/CRC32CDigestManager.java | 65 ++++++++
.../apache/bookkeeper/client/DigestManager.java | 2 +
.../apache/bookkeeper/client/LedgerMetadata.java | 15 +-
.../apache/bookkeeper/client/api/DigestType.java | 6 +-
...BookieWriteLedgersWithDifferentDigestsTest.java | 28 ++--
.../main/resources/bookkeeper/findbugsExclude.xml | 4 +
microbenchmarks/pom.xml | 158 +++++++++++++++++++
microbenchmarks/results.txt | 106 +++++++++++++
microbenchmarks/run.sh | 19 +++
.../bookkeeper/client/DigestTypeBenchmark.java | 173 +++++++++++++++++++++
pom.xml | 1 +
14 files changed, 569 insertions(+), 27 deletions(-)
diff --git a/bookkeeper-proto/src/main/proto/DataFormats.proto b/bookkeeper-proto/src/main/proto/DataFormats.proto
index cdade95..5a5d65f 100644
--- a/bookkeeper-proto/src/main/proto/DataFormats.proto
+++ b/bookkeeper-proto/src/main/proto/DataFormats.proto
@@ -45,6 +45,7 @@ message LedgerMetadataFormat {
enum DigestType {
CRC32 = 1;
HMAC = 2;
+ CRC32C = 3;
}
optional DigestType digestType = 7;
optional bytes password = 8;
diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml
index 8d3a052..5b82066 100644
--- a/bookkeeper-server/pom.xml
+++ b/bookkeeper-server/pom.xml
@@ -182,6 +182,11 @@
<artifactId>http-server</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-checksum</artifactId>
+ <version>1.20.0-incubating</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
index 17df48c..6dddfb1 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
@@ -636,10 +636,19 @@ public class BookKeeper implements org.apache.bookkeeper.client.api.BookKeeper {
* report fake bytes with a mathching MAC unless it knows the password
*/
public enum DigestType {
- MAC, CRC32;
+ MAC, CRC32, CRC32C;
public static DigestType fromApiDigestType(org.apache.bookkeeper.client.api.DigestType digestType) {
- return digestType == org.apache.bookkeeper.client.api.DigestType.MAC ? MAC : CRC32;
+ switch (digestType) {
+ case MAC:
+ return DigestType.MAC;
+ case CRC32:
+ return DigestType.CRC32;
+ case CRC32C:
+ return DigestType.CRC32C;
+ default:
+ throw new IllegalArgumentException("Unable to convert digest type " + digestType);
+ }
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
new file mode 100644
index 0000000..1923220
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
@@ -0,0 +1,65 @@
+package org.apache.bookkeeper.client;
+
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+import com.scurrilous.circe.crc.Sse42Crc32C;
+import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.pulsar.checksum.utils.Crc32cChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class CRC32CDigestManager extends DigestManager {
+ static final Logger LOG = LoggerFactory.getLogger(CRC32CDigestManager.class);
+
+ private final ThreadLocal<MutableInt> currentCrc = ThreadLocal
+ .withInitial(() -> new MutableInt(0));
+ private final ThreadLocal<MutableBoolean> isNewCrc = ThreadLocal
+ .withInitial(() -> new MutableBoolean(true));
+
+ public CRC32CDigestManager(long ledgerId) {
+ super(ledgerId);
+ if (!Sse42Crc32C.isSupported()) {
+ LOG.error("Sse42Crc32C is not supported, will use less slower CRC32C implementation.");
+ }
+ }
+
+ @Override
+ int getMacCodeLength() {
+ return 4;
+ }
+
+ @Override
+ void populateValueAndReset(ByteBuf buf) {
+ buf.writeInt(currentCrc.get().intValue());
+ isNewCrc.get().setTrue();
+ }
+
+ @Override
+ void update(ByteBuf data) {
+ if (isNewCrc.get().isTrue()) {
+ isNewCrc.get().setFalse();
+ currentCrc.get().setValue(Crc32cChecksum.computeChecksum(data));
+ } else {
+ final int lastCrc = currentCrc.get().intValue();
+ currentCrc.get().setValue(Crc32cChecksum.resumeChecksum(lastCrc, data));
+ }
+ }
+}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
index a5c734c..21c9ae7 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
@@ -69,6 +69,8 @@ abstract class DigestManager {
return new MacDigestManager(ledgerId, passwd);
case CRC32:
return new CRC32DigestManager(ledgerId);
+ case CRC32C:
+ return new CRC32CDigestManager(ledgerId);
default:
throw new GeneralSecurityException("Unknown checksum type: " + digestType);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index 62ca283..d37e7fc 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -119,7 +119,7 @@ public class LedgerMetadata implements org.apache.bookkeeper.client.api.LedgerMe
this.metadataFormatVersion = CURRENT_METADATA_FORMAT_VERSION;
this.digestType = digestType.equals(BookKeeper.DigestType.MAC)
- ? LedgerMetadataFormat.DigestType.HMAC : LedgerMetadataFormat.DigestType.CRC32;
+ ? LedgerMetadataFormat.DigestType.HMAC : LedgerMetadataFormat.DigestType.valueOf(digestType.toString());
this.password = Arrays.copyOf(password, password.length);
this.hasPassword = true;
if (customMetadata != null) {
@@ -230,10 +230,15 @@ public class LedgerMetadata implements org.apache.bookkeeper.client.api.LedgerMe
@Override
public DigestType getDigestType() {
- if (digestType.equals(LedgerMetadataFormat.DigestType.HMAC)) {
- return DigestType.MAC;
- } else {
- return DigestType.CRC32;
+ switch (digestType) {
+ case HMAC:
+ return DigestType.MAC;
+ case CRC32:
+ return DigestType.CRC32;
+ case CRC32C:
+ return DigestType.CRC32C;
+ default:
+ throw new IllegalArgumentException("Unable to convert digest type " + digestType);
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
index d7db214..728a246 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
@@ -39,5 +39,9 @@ public enum DigestType {
/**
* Entries are verified by applied MAC algorithm.
*/
- MAC
+ MAC,
+ /**
+ * Entries are verified by applied CRC32C algorithm.
+ */
+ CRC32C
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
index 3e069ca..daa434c 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
@@ -54,7 +54,7 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
.getLogger(BookieWriteLedgersWithDifferentDigestsTest.class);
byte[] ledgerPassword = "aaa".getBytes();
- LedgerHandle lh, lh2;
+ LedgerHandle lh;
Enumeration<LedgerEntry> ls;
// test related variables
@@ -65,6 +65,7 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
ArrayList<byte[]> entries2; // generated entries
private final DigestType digestType;
+ private final DigestType otherDigestType;
private static class SyncObj {
volatile int counter;
@@ -77,7 +78,7 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { {DigestType.MAC }, {DigestType.CRC32}});
+ return Arrays.asList(new Object[][] { {DigestType.MAC }, {DigestType.CRC32}, {DigestType.CRC32C} });
}
@Override
@@ -93,6 +94,7 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
public BookieWriteLedgersWithDifferentDigestsTest(DigestType digestType) {
super(3);
this.digestType = digestType;
+ this.otherDigestType = digestType == DigestType.CRC32 ? DigestType.MAC : DigestType.CRC32;
String ledgerManagerFactory = "org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory";
// set ledger manager
baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
@@ -103,11 +105,11 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
public void testLedgersWithDifferentDigestTypesNoAutodetection() throws Exception {
bkc.conf.setEnableDigestTypeAutodetection(false);
// Create ledgers
- lh = bkc.createLedgerAdv(3, 2, 2, DigestType.MAC, ledgerPassword);
+ lh = bkc.createLedgerAdv(3, 2, 2, digestType, ledgerPassword);
final long id = lh.ledgerId;
- LOG.info("Ledger ID-1: " + lh.getId());
+ LOG.info("Ledger ID: {}, digestType: {}", lh.getId(), digestType);
SyncObj syncObj1 = new SyncObj();
for (int i = numEntriesToWrite - 1; i >= 0; i--) {
ByteBuffer entry = ByteBuffer.allocate(4);
@@ -125,7 +127,7 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
lh.close();
try {
- bkc.openLedgerNoRecovery(id, DigestType.CRC32, ledgerPassword).close();
+ bkc.openLedgerNoRecovery(id, otherDigestType, ledgerPassword).close();
fail("digest mismatch error is expected");
} catch (BKException bke) {
// expected
@@ -136,44 +138,32 @@ public class BookieWriteLedgersWithDifferentDigestsTest extends
public void testLedgersWithDifferentDigestTypesWithAutodetection() throws Exception {
bkc.conf.setEnableDigestTypeAutodetection(true);
// Create ledgers
- lh = bkc.createLedgerAdv(3, 2, 2, DigestType.MAC, ledgerPassword);
- lh2 = bkc.createLedgerAdv(3, 2, 2, DigestType.CRC32, ledgerPassword);
+ lh = bkc.createLedgerAdv(3, 2, 2, digestType, ledgerPassword);
final long id = lh.ledgerId;
- final long id2 = lh2.ledgerId;
LOG.info("Ledger ID-1: " + lh.getId());
- LOG.info("Ledger ID-2: " + lh2.getId());
SyncObj syncObj1 = new SyncObj();
- SyncObj syncObj2 = new SyncObj();
for (int i = numEntriesToWrite - 1; i >= 0; i--) {
ByteBuffer entry = ByteBuffer.allocate(4);
entry.putInt(rng.nextInt(maxInt));
entry.position(0);
entries1.add(0, entry.array());
- entries2.add(0, entry.array());
lh.asyncAddEntry(i, entry.array(), 0, entry.capacity(), this, syncObj1);
- lh2.asyncAddEntry(i, entry.array(), 0, entry.capacity(), this, syncObj2);
}
// Wait for all entries to be acknowledged
waitForEntriesAddition(syncObj1, numEntriesToWrite);
- waitForEntriesAddition(syncObj2, numEntriesToWrite);
// Reads here work ok because ledger uses digest type set during create
readEntries(lh, entries1);
- readEntries(lh2, entries2);
lh.close();
- lh2.close();
// open here would fail if provided digest type is used
// it passes because ledger just uses digest type from its metadata/autodetects it
- lh = bkc.openLedgerNoRecovery(id, DigestType.CRC32, ledgerPassword);
- lh2 = bkc.openLedgerNoRecovery(id2, DigestType.MAC, ledgerPassword);
+ lh = bkc.openLedgerNoRecovery(id, otherDigestType, ledgerPassword);
readEntries(lh, entries1);
- readEntries(lh2, entries2);
lh.close();
- lh2.close();
}
private void waitForEntriesAddition(SyncObj syncObj, int numEntriesToWrite) throws InterruptedException {
diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
index 736c88b..4410382 100644
--- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
+++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
@@ -33,6 +33,10 @@
<Class name="~org\.apache\.bookkeeper\.tests\.generated.*" />
</Match>
<Match>
+ <!-- generated code -->
+ <Class name="~org\.apache\.bookkeeper\.client\.generated.*"/>
+ </Match>
+ <Match>
<!-- it is safe to store external bytes reference here. since we are using
bytes from a slab. //-->
<Class name="org.apache.bookkeeper.bookie.EntryKeyValue" />
diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml
new file mode 100644
index 0000000..4bdc789
--- /dev/null
+++ b/microbenchmarks/pom.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper</artifactId>
+ <version>4.7.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>microbenchmarks</artifactId>
+ <name>Apache BookKeeper :: microbenchmarks</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <jmh.version>1.19</jmh.version>
+ <javac.target>1.8</javac.target>
+ <uberjar.name>benchmarks</uberjar.name>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>${netty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-server</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>compile</scope>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <compilerVersion>${javac.target}</compilerVersion>
+ <source>${javac.target}</source>
+ <target>${javac.target}</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${uberjar.name}</finalName>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <!-- Shading signed JARs will fail without
+ this. http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.17</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
diff --git a/microbenchmarks/results.txt b/microbenchmarks/results.txt
new file mode 100644
index 0000000..ed2e522
--- /dev/null
+++ b/microbenchmarks/results.txt
@@ -0,0 +1,106 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# on macbook pro, for 64KB entry
+# Run complete. Total time: 00:21:26
+
+Benchmark (bufferType) (digest) Mode Cnt Score Error Units
+DigestTypeBenchmark.digestManager ARRAY_BACKED MAC thrpt 5 8.586 ± 0.556 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate ARRAY_BACKED MAC thrpt 5 0.692 ± 0.045 MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm ARRAY_BACKED MAC thrpt 5 88.010 ± 0.002 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space ARRAY_BACKED MAC thrpt 5 1.615 ± 8.515 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm ARRAY_BACKED MAC thrpt 5 203.670 ± 1073.904 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Survivor_Space ARRAY_BACKED MAC thrpt 5 0.096 ± 0.826 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Survivor_Space.norm ARRAY_BACKED MAC thrpt 5 12.176 ± 104.841 B/op
+DigestTypeBenchmark.digestManager:·gc.count ARRAY_BACKED MAC thrpt 5 2.000 counts
+DigestTypeBenchmark.digestManager:·gc.time ARRAY_BACKED MAC thrpt 5 16.000 ms
+DigestTypeBenchmark.digestManager:·stack ARRAY_BACKED MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager ARRAY_BACKED CRC32 thrpt 5 445.869 ± 8.161 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate ARRAY_BACKED CRC32 thrpt 5 ≈ 10⁻⁴ MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm ARRAY_BACKED CRC32 thrpt 5 ≈ 10⁻⁴ B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space ARRAY_BACKED CRC32 thrpt 5 0.815 ± 7.022 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm ARRAY_BACKED CRC32 thrpt 5 1.997 ± 17.191 B/op
+DigestTypeBenchmark.digestManager:·gc.count ARRAY_BACKED CRC32 thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:·gc.time ARRAY_BACKED CRC32 thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:·stack ARRAY_BACKED CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager ARRAY_BACKED CRC32_C thrpt 5 832.499 ± 56.604 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate ARRAY_BACKED CRC32_C thrpt 5 ≈ 10⁻⁴ MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm ARRAY_BACKED CRC32_C thrpt 5 ≈ 10⁻⁴ B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space ARRAY_BACKED CRC32_C thrpt 5 0.816 ± 7.023 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm ARRAY_BACKED CRC32_C thrpt 5 1.064 ± 9.158 B/op
+DigestTypeBenchmark.digestManager:·gc.count ARRAY_BACKED CRC32_C thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:·gc.time ARRAY_BACKED CRC32_C thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:·stack ARRAY_BACKED CRC32_C thrpt NaN ---
+DigestTypeBenchmark.digestManager NOT_ARRAY_BACKED MAC thrpt 5 8.367 ± 0.045 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate NOT_ARRAY_BACKED MAC thrpt 5 502.288 ± 2.868 MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm NOT_ARRAY_BACKED MAC thrpt 5 65593.694 ± 12.966 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space NOT_ARRAY_BACKED MAC thrpt 5 539.102 ± 0.215 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm NOT_ARRAY_BACKED MAC thrpt 5 70401.331 ± 403.899 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen NOT_ARRAY_BACKED MAC thrpt 5 0.006 ± 0.015 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen.norm NOT_ARRAY_BACKED MAC thrpt 5 0.784 ± 1.962 B/op
+DigestTypeBenchmark.digestManager:·gc.count NOT_ARRAY_BACKED MAC thrpt 5 55.000 counts
+DigestTypeBenchmark.digestManager:·gc.time NOT_ARRAY_BACKED MAC thrpt 5 200.000 ms
+DigestTypeBenchmark.digestManager:·stack NOT_ARRAY_BACKED MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager NOT_ARRAY_BACKED CRC32 thrpt 5 119.933 ± 7.129 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate NOT_ARRAY_BACKED CRC32 thrpt 5 7206.443 ± 430.557 MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm NOT_ARRAY_BACKED CRC32 thrpt 5 65648.001 ± 0.001 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space NOT_ARRAY_BACKED CRC32 thrpt 5 7684.923 ± 509.880 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm NOT_ARRAY_BACKED CRC32 thrpt 5 70005.330 ± 532.187 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen NOT_ARRAY_BACKED CRC32 thrpt 5 0.158 ± 0.078 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen.norm NOT_ARRAY_BACKED CRC32 thrpt 5 1.439 ± 0.769 B/op
+DigestTypeBenchmark.digestManager:·gc.count NOT_ARRAY_BACKED CRC32 thrpt 5 784.000 counts
+DigestTypeBenchmark.digestManager:·gc.time NOT_ARRAY_BACKED CRC32 thrpt 5 2949.000 ms
+DigestTypeBenchmark.digestManager:·stack NOT_ARRAY_BACKED CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager NOT_ARRAY_BACKED CRC32_C thrpt 5 130.444 ± 3.441 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate NOT_ARRAY_BACKED CRC32_C thrpt 5 7824.727 ± 212.028 MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm NOT_ARRAY_BACKED CRC32_C thrpt 5 65552.001 ± 0.001 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space NOT_ARRAY_BACKED CRC32_C thrpt 5 8349.266 ± 177.504 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm NOT_ARRAY_BACKED CRC32_C thrpt 5 69947.075 ± 707.700 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen NOT_ARRAY_BACKED CRC32_C thrpt 5 0.129 ± 0.083 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen.norm NOT_ARRAY_BACKED CRC32_C thrpt 5 1.079 ± 0.691 B/op
+DigestTypeBenchmark.digestManager:·gc.count NOT_ARRAY_BACKED CRC32_C thrpt 5 852.000 counts
+DigestTypeBenchmark.digestManager:·gc.time NOT_ARRAY_BACKED CRC32_C thrpt 5 3229.000 ms
+DigestTypeBenchmark.digestManager:·stack NOT_ARRAY_BACKED CRC32_C thrpt NaN ---
+DigestTypeBenchmark.digestManager BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 8.642 ± 0.132 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 1.329 ± 0.021 MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 168.010 ± 0.002 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 0.799 ± 6.880 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 101.377 ± 872.889 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Survivor_Space BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 0.096 ± 0.826 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Survivor_Space.norm BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 12.165 ± 104.747 B/op
+DigestTypeBenchmark.digestManager:·gc.count BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:·gc.time BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:·stack BYTE_BUF_DEFAULT_ALLOC MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 445.372 ± 14.813 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 26.083 ± 0.883 MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 64.000 ± 0.001 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 24.437 ± 93.861 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 60.215 ± 231.193 B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 0.002 ± 0.017 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Old_Gen.norm BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 0.005 ± 0.042 B/op
+DigestTypeBenchmark.digestManager:·gc.count BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 3.000 counts
+DigestTypeBenchmark.digestManager:·gc.time BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 18.000 ms
+DigestTypeBenchmark.digestManager:·stack BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 868.398 ± 9.782 ops/ms
+DigestTypeBenchmark.digestManager:·gc.alloc.rate BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 ≈ 10⁻⁴ MB/sec
+DigestTypeBenchmark.digestManager:·gc.alloc.rate.norm BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 ≈ 10⁻⁴ B/op
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 0.815 ± 7.018 MB/sec
+DigestTypeBenchmark.digestManager:·gc.churn.G1_Eden_Space.norm BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 1.023 ± 8.808 B/op
+DigestTypeBenchmark.digestManager:·gc.count BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:·gc.time BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 7.000 ms
+DigestTypeBenchmark.digestManager:·stack BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt NaN ---
+
diff --git a/microbenchmarks/run.sh b/microbenchmarks/run.sh
new file mode 100755
index 0000000..0d59321
--- /dev/null
+++ b/microbenchmarks/run.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+java -Xms1G -Xmx1G -Djdk.nio.maxCachedBufferSize=0 -Djava.net.preferIPv4Stack=true -Duser.timezone=UTC -XX:-MaxFDLimit -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ResizeTLAB -XX:-ResizePLAB -XX:MetaspaceSize=128m -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -XX:+ParallelRefProcEnabled -XX:StackShadowPages=20 -XX:+UseCompressedOops -XX:+DisableExplicitGC -XX:StringTableSize=1000003 -XX:InitiatingHeapOccupancyPercent=40 -jar target/benchmarks.jar -prof gc -prof stack:lines=5;ti [...]
+
diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
new file mode 100644
index 0000000..bda4da5
--- /dev/null
+++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
@@ -0,0 +1,173 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.bookkeeper.client;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBufAllocator;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.util.DoubleByteBuf;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+/**
+ * Microbenchmarks for different digest type
+ * getting started:
+ * 1. http://tutorials.jenkov.com/java-performance/jmh.html
+ * 2. http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/
+ * 3. google
+ *
+ * To run:
+ * build project from command line.
+ * execute ./run.sh
+ */
+public class DigestTypeBenchmark {
+
+ public enum BufferType {
+ ARRAY_BACKED,
+ NOT_ARRAY_BACKED,
+ BYTE_BUF_DEFAULT_ALLOC
+ }
+
+ public enum Digest {
+ MAC,
+ CRC32,
+ CRC32_C,
+ }
+
+ static byte[] randomBytes(int sz) {
+ byte[] b = new byte[sz];
+ ThreadLocalRandom.current().nextBytes(b);
+ return b;
+ }
+
+ @State(Scope.Thread)
+ public static class MyState {
+
+ @Param
+ public BufferType bufferType;
+ @Param
+ public Digest digest;
+ @Param({"1024", "4086", "8192", "16384", "65536"})
+ public int entrySize;
+
+ private DigestManager crc32;
+ private DigestManager crc32c;
+ private DigestManager mac;
+
+ private ByteBuf arrayBackedBuffer;
+ private ByteBuf notArrayBackedBuffer;
+ private ByteBuf byteBufDefaultAlloc;
+
+ public ByteBuf digestBuf;
+
+ @Setup(Level.Trial)
+ public void doSetup() throws Exception {
+ final byte[] password = "password".getBytes("UTF-8");
+ crc32 = DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE),
+ password, DigestType.CRC32);
+
+ crc32c = DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE),
+ password, DigestType.CRC32C);
+
+ mac = DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE),
+ password, DigestType.MAC);
+
+ digestBuf = Unpooled.buffer(getDigestManager(digest).getMacCodeLength());
+
+ arrayBackedBuffer = Unpooled.wrappedBuffer(randomBytes(entrySize));
+
+ final int headerSize = 32 + getDigestManager(digest).getMacCodeLength();
+ notArrayBackedBuffer = DoubleByteBuf.get(Unpooled.wrappedBuffer(randomBytes(headerSize)),
+ Unpooled.wrappedBuffer((randomBytes(entrySize - headerSize))));
+
+ byteBufDefaultAlloc = ByteBufAllocator.DEFAULT.buffer(entrySize, entrySize);
+ byteBufDefaultAlloc.writeBytes(randomBytes(entrySize));
+
+ if (!arrayBackedBuffer.hasArray() || notArrayBackedBuffer.hasArray()) {
+ throw new IllegalStateException("buffers in invalid state");
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void doTearDown() {
+ }
+
+ public ByteBuf getByteBuff(BufferType bType) {
+ switch (bType) {
+ case ARRAY_BACKED:
+ return arrayBackedBuffer;
+ case NOT_ARRAY_BACKED:
+ return notArrayBackedBuffer;
+ case BYTE_BUF_DEFAULT_ALLOC:
+ return byteBufDefaultAlloc;
+ default:
+ throw new IllegalArgumentException("unknown buffer type " + bType);
+ }
+ }
+
+ public DigestManager getDigestManager(Digest digest) {
+ switch (digest) {
+ case CRC32:
+ return crc32;
+ case CRC32_C:
+ return crc32c;
+ case MAC:
+ return mac;
+ default:
+ throw new IllegalArgumentException("unknown digest " + digest);
+ }
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ @Warmup(iterations = 2, time = 3, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 5, time = 12, timeUnit = TimeUnit.SECONDS)
+ @Threads(2)
+ @Fork(value = 1, warmups = 1)
+ public void digestManager(MyState state) {
+ final ByteBuf buff = state.getByteBuff(state.bufferType);
+ final DigestManager dm = state.getDigestManager(state.digest);
+ dm.update(buff);
+ state.digestBuf.clear();
+ dm.populateValueAndReset(state.digestBuf);
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 9ec92ab..b3f6b75 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,6 +62,7 @@
<module>shaded</module>
<module>tests</module>
<module>bookkeeper-dist</module>
+ <module>microbenchmarks</module>
</modules>
<mailingLists>
<mailingList>
--
To stop receiving notification emails like this one, please contact
['"commits@bookkeeper.apache.org" <co...@bookkeeper.apache.org>'].