You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2017/12/15 19:58:31 UTC
[GitHub] sijie closed pull request #856: ISSUE #590 (@bug W-4556980@) Reduce excessive CPU usage on client side: add crc3?
sijie closed pull request #856: ISSUE #590 (@bug W-4556980@) Reduce excessive CPU usage on client side: add crc3?
URL: https://github.com/apache/bookkeeper/pull/856
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-proto/src/main/proto/DataFormats.proto b/bookkeeper-proto/src/main/proto/DataFormats.proto
index cdade9563..5a5d65f79 100644
--- a/bookkeeper-proto/src/main/proto/DataFormats.proto
+++ b/bookkeeper-proto/src/main/proto/DataFormats.proto
@@ -45,6 +45,7 @@ message LedgerMetadataFormat {
enum DigestType {
CRC32 = 1;
HMAC = 2;
+ CRC32C = 3;
}
optional DigestType digestType = 7;
optional bytes password = 8;
diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml
index 8c057e6de..45e88eaf2 100644
--- a/bookkeeper-server/pom.xml
+++ b/bookkeeper-server/pom.xml
@@ -182,6 +182,11 @@
<artifactId>http-server</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-checksum</artifactId>
+ <version>1.20.0-incubating</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
index 69e113993..e0959f45b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
@@ -634,10 +634,19 @@ boolean isReorderReadSequence() {
* report fake bytes with a mathching MAC unless it knows the password
*/
public enum DigestType {
- MAC, CRC32;
+ MAC, CRC32, CRC32C;
public static DigestType fromApiDigestType(org.apache.bookkeeper.client.api.DigestType digestType) {
- return digestType == org.apache.bookkeeper.client.api.DigestType.MAC ? MAC : CRC32;
+ switch (digestType) {
+ case MAC:
+ return DigestType.MAC;
+ case CRC32:
+ return DigestType.CRC32;
+ case CRC32C:
+ return DigestType.CRC32C;
+ default:
+ throw new IllegalArgumentException("Unable to convert digest type " + digestType);
+ }
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
new file mode 100644
index 000000000..1923220d7
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
@@ -0,0 +1,65 @@
+package org.apache.bookkeeper.client;
+
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+import com.scurrilous.circe.crc.Sse42Crc32C;
+import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.pulsar.checksum.utils.Crc32cChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class CRC32CDigestManager extends DigestManager {
+ static final Logger LOG = LoggerFactory.getLogger(CRC32CDigestManager.class);
+
+ private final ThreadLocal<MutableInt> currentCrc = ThreadLocal
+ .withInitial(() -> new MutableInt(0));
+ private final ThreadLocal<MutableBoolean> isNewCrc = ThreadLocal
+ .withInitial(() -> new MutableBoolean(true));
+
+ public CRC32CDigestManager(long ledgerId) {
+ super(ledgerId);
+ if (!Sse42Crc32C.isSupported()) {
+ LOG.error("Sse42Crc32C is not supported, will use less slower CRC32C implementation.");
+ }
+ }
+
+ @Override
+ int getMacCodeLength() {
+ return 4;
+ }
+
+ @Override
+ void populateValueAndReset(ByteBuf buf) {
+ buf.writeInt(currentCrc.get().intValue());
+ isNewCrc.get().setTrue();
+ }
+
+ @Override
+ void update(ByteBuf data) {
+ if (isNewCrc.get().isTrue()) {
+ isNewCrc.get().setFalse();
+ currentCrc.get().setValue(Crc32cChecksum.computeChecksum(data));
+ } else {
+ final int lastCrc = currentCrc.get().intValue();
+ currentCrc.get().setValue(Crc32cChecksum.resumeChecksum(lastCrc, data));
+ }
+ }
+}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
index a5c734c51..21c9ae7a4 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
@@ -69,6 +69,8 @@ static DigestManager instantiate(long ledgerId, byte[] passwd, DigestType digest
return new MacDigestManager(ledgerId, passwd);
case CRC32:
return new CRC32DigestManager(ledgerId);
+ case CRC32C:
+ return new CRC32CDigestManager(ledgerId);
default:
throw new GeneralSecurityException("Unknown checksum type: " + digestType);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index 62ca2833c..d37e7fc80 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -119,7 +119,7 @@ public LedgerMetadata(int ensembleSize,
this.metadataFormatVersion = CURRENT_METADATA_FORMAT_VERSION;
this.digestType = digestType.equals(BookKeeper.DigestType.MAC)
- ? LedgerMetadataFormat.DigestType.HMAC : LedgerMetadataFormat.DigestType.CRC32;
+ ? LedgerMetadataFormat.DigestType.HMAC : LedgerMetadataFormat.DigestType.valueOf(digestType.toString());
this.password = Arrays.copyOf(password, password.length);
this.hasPassword = true;
if (customMetadata != null) {
@@ -230,10 +230,15 @@ boolean hasPassword() {
@Override
public DigestType getDigestType() {
- if (digestType.equals(LedgerMetadataFormat.DigestType.HMAC)) {
- return DigestType.MAC;
- } else {
- return DigestType.CRC32;
+ switch (digestType) {
+ case HMAC:
+ return DigestType.MAC;
+ case CRC32:
+ return DigestType.CRC32;
+ case CRC32C:
+ return DigestType.CRC32C;
+ default:
+ throw new IllegalArgumentException("Unable to convert digest type " + digestType);
}
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
index d7db214b0..728a24691 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
@@ -39,5 +39,9 @@
/**
* Entries are verified by applied MAC algorithm.
*/
- MAC
+ MAC,
+ /**
+ * Entries are verified by applied CRC32C algorithm.
+ */
+ CRC32C
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
index 3e069ca00..2100451ab 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
@@ -54,7 +54,7 @@
.getLogger(BookieWriteLedgersWithDifferentDigestsTest.class);
byte[] ledgerPassword = "aaa".getBytes();
- LedgerHandle lh, lh2;
+ LedgerHandle lh;
Enumeration<LedgerEntry> ls;
// test related variables
@@ -65,6 +65,7 @@
ArrayList<byte[]> entries2; // generated entries
private final DigestType digestType;
+ private final DigestType otherDigestType;
private static class SyncObj {
volatile int counter;
@@ -77,7 +78,7 @@ public SyncObj() {
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { {DigestType.MAC }, {DigestType.CRC32}});
+ return Arrays.asList(new Object[][] { {DigestType.MAC }, {DigestType.CRC32}, {DigestType.CRC32C} });
}
@Override
@@ -93,21 +94,22 @@ public void setUp() throws Exception {
public BookieWriteLedgersWithDifferentDigestsTest(DigestType digestType) {
super(3);
this.digestType = digestType;
+ this.otherDigestType = digestType == DigestType.CRC32 ? DigestType.MAC : DigestType.CRC32;
String ledgerManagerFactory = "org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory";
// set ledger manager
baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
}
- @Test
+ @Test()
public void testLedgersWithDifferentDigestTypesNoAutodetection() throws Exception {
bkc.conf.setEnableDigestTypeAutodetection(false);
// Create ledgers
- lh = bkc.createLedgerAdv(3, 2, 2, DigestType.MAC, ledgerPassword);
+ lh = bkc.createLedgerAdv(3, 2, 2, digestType, ledgerPassword);
final long id = lh.ledgerId;
- LOG.info("Ledger ID-1: " + lh.getId());
+ LOG.info("Ledger ID: {}, digestType: {}", lh.getId(), digestType);
SyncObj syncObj1 = new SyncObj();
for (int i = numEntriesToWrite - 1; i >= 0; i--) {
ByteBuffer entry = ByteBuffer.allocate(4);
@@ -125,7 +127,7 @@ public void testLedgersWithDifferentDigestTypesNoAutodetection() throws Exceptio
lh.close();
try {
- bkc.openLedgerNoRecovery(id, DigestType.CRC32, ledgerPassword).close();
+ bkc.openLedgerNoRecovery(id, otherDigestType, ledgerPassword).close();
fail("digest mismatch error is expected");
} catch (BKException bke) {
// expected
@@ -136,44 +138,32 @@ public void testLedgersWithDifferentDigestTypesNoAutodetection() throws Exceptio
public void testLedgersWithDifferentDigestTypesWithAutodetection() throws Exception {
bkc.conf.setEnableDigestTypeAutodetection(true);
// Create ledgers
- lh = bkc.createLedgerAdv(3, 2, 2, DigestType.MAC, ledgerPassword);
- lh2 = bkc.createLedgerAdv(3, 2, 2, DigestType.CRC32, ledgerPassword);
+ lh = bkc.createLedgerAdv(3, 2, 2, digestType, ledgerPassword);
final long id = lh.ledgerId;
- final long id2 = lh2.ledgerId;
LOG.info("Ledger ID-1: " + lh.getId());
- LOG.info("Ledger ID-2: " + lh2.getId());
SyncObj syncObj1 = new SyncObj();
- SyncObj syncObj2 = new SyncObj();
for (int i = numEntriesToWrite - 1; i >= 0; i--) {
ByteBuffer entry = ByteBuffer.allocate(4);
entry.putInt(rng.nextInt(maxInt));
entry.position(0);
entries1.add(0, entry.array());
- entries2.add(0, entry.array());
lh.asyncAddEntry(i, entry.array(), 0, entry.capacity(), this, syncObj1);
- lh2.asyncAddEntry(i, entry.array(), 0, entry.capacity(), this, syncObj2);
}
// Wait for all entries to be acknowledged
waitForEntriesAddition(syncObj1, numEntriesToWrite);
- waitForEntriesAddition(syncObj2, numEntriesToWrite);
// Reads here work ok because ledger uses digest type set during create
readEntries(lh, entries1);
- readEntries(lh2, entries2);
lh.close();
- lh2.close();
// open here would fail if provided digest type is used
// it passes because ledger just uses digest type from its metadata/autodetects it
- lh = bkc.openLedgerNoRecovery(id, DigestType.CRC32, ledgerPassword);
- lh2 = bkc.openLedgerNoRecovery(id2, DigestType.MAC, ledgerPassword);
+ lh = bkc.openLedgerNoRecovery(id, otherDigestType, ledgerPassword);
readEntries(lh, entries1);
- readEntries(lh2, entries2);
lh.close();
- lh2.close();
}
private void waitForEntriesAddition(SyncObj syncObj, int numEntriesToWrite) throws InterruptedException {
diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
index 2cc5ce766..866c946d6 100644
--- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
+++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
@@ -28,6 +28,10 @@
<!-- generated code, we can't be held responsible for findbugs in it //-->
<Class name="~org\.apache\.bookkeeper\.tests\.generated.*" />
</Match>
+ <Match>
+ <!-- generated code -->
+ <Class name="~org\.apache\.bookkeeper\.client\.generated.*"/>
+ </Match>
<Match>
<!-- it is safe to store external bytes reference here. since we are using
bytes from a slab. //-->
diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml
new file mode 100644
index 000000000..4bdc7897e
--- /dev/null
+++ b/microbenchmarks/pom.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper</artifactId>
+ <version>4.7.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>microbenchmarks</artifactId>
+ <name>Apache BookKeeper :: microbenchmarks</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <jmh.version>1.19</jmh.version>
+ <javac.target>1.8</javac.target>
+ <uberjar.name>benchmarks</uberjar.name>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>${netty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-server</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>compile</scope>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <compilerVersion>${javac.target}</compilerVersion>
+ <source>${javac.target}</source>
+ <target>${javac.target}</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${uberjar.name}</finalName>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <!-- Shading signed JARs will fail without
+ this. http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.17</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
diff --git a/microbenchmarks/results.txt b/microbenchmarks/results.txt
new file mode 100644
index 000000000..e9438aea1
--- /dev/null
+++ b/microbenchmarks/results.txt
@@ -0,0 +1,89 @@
+# on macbook pro, for 64KB entry
+# Run complete. Total time: 00:21:26
+
+Benchmark (bufferType) (digest) Mode Cnt Score Error Units
+DigestTypeBenchmark.digestManager ARRAY_BACKED MAC thrpt 5 8.586 ? 0.556 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate ARRAY_BACKED MAC thrpt 5 0.692 ? 0.045 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm ARRAY_BACKED MAC thrpt 5 88.010 ? 0.002 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space ARRAY_BACKED MAC thrpt 5 1.615 ? 8.515 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm ARRAY_BACKED MAC thrpt 5 203.670 ? 1073.904 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space ARRAY_BACKED MAC thrpt 5 0.096 ? 0.826 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space.norm ARRAY_BACKED MAC thrpt 5 12.176 ? 104.841 B/op
+DigestTypeBenchmark.digestManager:?gc.count ARRAY_BACKED MAC thrpt 5 2.000 counts
+DigestTypeBenchmark.digestManager:?gc.time ARRAY_BACKED MAC thrpt 5 16.000 ms
+DigestTypeBenchmark.digestManager:?stack ARRAY_BACKED MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager ARRAY_BACKED CRC32 thrpt 5 445.869 ? 8.161 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate ARRAY_BACKED CRC32 thrpt 5 ? 10?? MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm ARRAY_BACKED CRC32 thrpt 5 ? 10?? B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space ARRAY_BACKED CRC32 thrpt 5 0.815 ? 7.022 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm ARRAY_BACKED CRC32 thrpt 5 1.997 ? 17.191 B/op
+DigestTypeBenchmark.digestManager:?gc.count ARRAY_BACKED CRC32 thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time ARRAY_BACKED CRC32 thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:?stack ARRAY_BACKED CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager ARRAY_BACKED CRC32_C thrpt 5 832.499 ? 56.604 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate ARRAY_BACKED CRC32_C thrpt 5 ? 10?? MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm ARRAY_BACKED CRC32_C thrpt 5 ? 10?? B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space ARRAY_BACKED CRC32_C thrpt 5 0.816 ? 7.023 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm ARRAY_BACKED CRC32_C thrpt 5 1.064 ? 9.158 B/op
+DigestTypeBenchmark.digestManager:?gc.count ARRAY_BACKED CRC32_C thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time ARRAY_BACKED CRC32_C thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:?stack ARRAY_BACKED CRC32_C thrpt NaN ---
+DigestTypeBenchmark.digestManager NOT_ARRAY_BACKED MAC thrpt 5 8.367 ? 0.045 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate NOT_ARRAY_BACKED MAC thrpt 5 502.288 ? 2.868 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm NOT_ARRAY_BACKED MAC thrpt 5 65593.694 ? 12.966 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space NOT_ARRAY_BACKED MAC thrpt 5 539.102 ? 0.215 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm NOT_ARRAY_BACKED MAC thrpt 5 70401.331 ? 403.899 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen NOT_ARRAY_BACKED MAC thrpt 5 0.006 ? 0.015 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm NOT_ARRAY_BACKED MAC thrpt 5 0.784 ? 1.962 B/op
+DigestTypeBenchmark.digestManager:?gc.count NOT_ARRAY_BACKED MAC thrpt 5 55.000 counts
+DigestTypeBenchmark.digestManager:?gc.time NOT_ARRAY_BACKED MAC thrpt 5 200.000 ms
+DigestTypeBenchmark.digestManager:?stack NOT_ARRAY_BACKED MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager NOT_ARRAY_BACKED CRC32 thrpt 5 119.933 ? 7.129 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate NOT_ARRAY_BACKED CRC32 thrpt 5 7206.443 ? 430.557 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm NOT_ARRAY_BACKED CRC32 thrpt 5 65648.001 ? 0.001 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space NOT_ARRAY_BACKED CRC32 thrpt 5 7684.923 ? 509.880 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm NOT_ARRAY_BACKED CRC32 thrpt 5 70005.330 ? 532.187 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen NOT_ARRAY_BACKED CRC32 thrpt 5 0.158 ? 0.078 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm NOT_ARRAY_BACKED CRC32 thrpt 5 1.439 ? 0.769 B/op
+DigestTypeBenchmark.digestManager:?gc.count NOT_ARRAY_BACKED CRC32 thrpt 5 784.000 counts
+DigestTypeBenchmark.digestManager:?gc.time NOT_ARRAY_BACKED CRC32 thrpt 5 2949.000 ms
+DigestTypeBenchmark.digestManager:?stack NOT_ARRAY_BACKED CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager NOT_ARRAY_BACKED CRC32_C thrpt 5 130.444 ? 3.441 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate NOT_ARRAY_BACKED CRC32_C thrpt 5 7824.727 ? 212.028 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm NOT_ARRAY_BACKED CRC32_C thrpt 5 65552.001 ? 0.001 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space NOT_ARRAY_BACKED CRC32_C thrpt 5 8349.266 ? 177.504 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm NOT_ARRAY_BACKED CRC32_C thrpt 5 69947.075 ? 707.700 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen NOT_ARRAY_BACKED CRC32_C thrpt 5 0.129 ? 0.083 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm NOT_ARRAY_BACKED CRC32_C thrpt 5 1.079 ? 0.691 B/op
+DigestTypeBenchmark.digestManager:?gc.count NOT_ARRAY_BACKED CRC32_C thrpt 5 852.000 counts
+DigestTypeBenchmark.digestManager:?gc.time NOT_ARRAY_BACKED CRC32_C thrpt 5 3229.000 ms
+DigestTypeBenchmark.digestManager:?stack NOT_ARRAY_BACKED CRC32_C thrpt NaN ---
+DigestTypeBenchmark.digestManager BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 8.642 ? 0.132 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 1.329 ? 0.021 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 168.010 ? 0.002 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 0.799 ? 6.880 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 101.377 ? 872.889 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 0.096 ? 0.826 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space.norm BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 12.165 ? 104.747 B/op
+DigestTypeBenchmark.digestManager:?gc.count BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:?stack BYTE_BUF_DEFAULT_ALLOC MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 445.372 ? 14.813 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 26.083 ? 0.883 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 64.000 ? 0.001 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 24.437 ? 93.861 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 60.215 ? 231.193 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 0.002 ? 0.017 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 0.005 ? 0.042 B/op
+DigestTypeBenchmark.digestManager:?gc.count BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 3.000 counts
+DigestTypeBenchmark.digestManager:?gc.time BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 18.000 ms
+DigestTypeBenchmark.digestManager:?stack BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 868.398 ? 9.782 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 ? 10?? MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 ? 10?? B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 0.815 ? 7.018 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 1.023 ? 8.808 B/op
+DigestTypeBenchmark.digestManager:?gc.count BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 7.000 ms
+DigestTypeBenchmark.digestManager:?stack BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt NaN ---
+
diff --git a/microbenchmarks/run.sh b/microbenchmarks/run.sh
new file mode 100755
index 000000000..0d59321ee
--- /dev/null
+++ b/microbenchmarks/run.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+java -Xms1G -Xmx1G -Djdk.nio.maxCachedBufferSize=0 -Djava.net.preferIPv4Stack=true -Duser.timezone=UTC -XX:-MaxFDLimit -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ResizeTLAB -XX:-ResizePLAB -XX:MetaspaceSize=128m -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -XX:+ParallelRefProcEnabled -XX:StackShadowPages=20 -XX:+UseCompressedOops -XX:+DisableExplicitGC -XX:StringTableSize=1000003 -XX:InitiatingHeapOccupancyPercent=40 -jar target/benchmarks.jar -prof gc -prof stack:lines=5;time=1;top=3
+
diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
new file mode 100644
index 000000000..bda4da57d
--- /dev/null
+++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
@@ -0,0 +1,173 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.bookkeeper.client;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBufAllocator;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.util.DoubleByteBuf;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+/**
+ * Microbenchmarks for different digest type
+ * getting started:
+ * 1. http://tutorials.jenkov.com/java-performance/jmh.html
+ * 2. http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/
+ * 3. google
+ *
+ * To run:
+ * build project from command line.
+ * execute ./run.sh
+ */
+public class DigestTypeBenchmark {
+
+ public enum BufferType {
+ ARRAY_BACKED,
+ NOT_ARRAY_BACKED,
+ BYTE_BUF_DEFAULT_ALLOC
+ }
+
+ public enum Digest {
+ MAC,
+ CRC32,
+ CRC32_C,
+ }
+
+ static byte[] randomBytes(int sz) {
+ byte[] b = new byte[sz];
+ ThreadLocalRandom.current().nextBytes(b);
+ return b;
+ }
+
+ @State(Scope.Thread)
+ public static class MyState {
+
+ @Param
+ public BufferType bufferType;
+ @Param
+ public Digest digest;
+ @Param({"1024", "4086", "8192", "16384", "65536"})
+ public int entrySize;
+
+ private DigestManager crc32;
+ private DigestManager crc32c;
+ private DigestManager mac;
+
+ private ByteBuf arrayBackedBuffer;
+ private ByteBuf notArrayBackedBuffer;
+ private ByteBuf byteBufDefaultAlloc;
+
+ public ByteBuf digestBuf;
+
+ @Setup(Level.Trial)
+ public void doSetup() throws Exception {
+ final byte[] password = "password".getBytes("UTF-8");
+ crc32 = DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE),
+ password, DigestType.CRC32);
+
+ crc32c = DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE),
+ password, DigestType.CRC32C);
+
+ mac = DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0, Long.MAX_VALUE),
+ password, DigestType.MAC);
+
+ digestBuf = Unpooled.buffer(getDigestManager(digest).getMacCodeLength());
+
+ arrayBackedBuffer = Unpooled.wrappedBuffer(randomBytes(entrySize));
+
+ final int headerSize = 32 + getDigestManager(digest).getMacCodeLength();
+ notArrayBackedBuffer = DoubleByteBuf.get(Unpooled.wrappedBuffer(randomBytes(headerSize)),
+ Unpooled.wrappedBuffer((randomBytes(entrySize - headerSize))));
+
+ byteBufDefaultAlloc = ByteBufAllocator.DEFAULT.buffer(entrySize, entrySize);
+ byteBufDefaultAlloc.writeBytes(randomBytes(entrySize));
+
+ if (!arrayBackedBuffer.hasArray() || notArrayBackedBuffer.hasArray()) {
+ throw new IllegalStateException("buffers in invalid state");
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void doTearDown() {
+ }
+
+ public ByteBuf getByteBuff(BufferType bType) {
+ switch (bType) {
+ case ARRAY_BACKED:
+ return arrayBackedBuffer;
+ case NOT_ARRAY_BACKED:
+ return notArrayBackedBuffer;
+ case BYTE_BUF_DEFAULT_ALLOC:
+ return byteBufDefaultAlloc;
+ default:
+ throw new IllegalArgumentException("unknown buffer type " + bType);
+ }
+ }
+
+ public DigestManager getDigestManager(Digest digest) {
+ switch (digest) {
+ case CRC32:
+ return crc32;
+ case CRC32_C:
+ return crc32c;
+ case MAC:
+ return mac;
+ default:
+ throw new IllegalArgumentException("unknown digest " + digest);
+ }
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ @Warmup(iterations = 2, time = 3, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 5, time = 12, timeUnit = TimeUnit.SECONDS)
+ @Threads(2)
+ @Fork(value = 1, warmups = 1)
+ public void digestManager(MyState state) {
+ final ByteBuf buff = state.getByteBuff(state.bufferType);
+ final DigestManager dm = state.getDigestManager(state.digest);
+ dm.update(buff);
+ state.digestBuf.clear();
+ dm.populateValueAndReset(state.digestBuf);
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 9c7a00270..0b15a8c16 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,6 +61,7 @@
<module>bookkeeper-http</module>
<module>tests</module>
<module>bookkeeper-dist</module>
+ <module>microbenchmarks</module>
</modules>
<mailingLists>
<mailingList>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services