You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2018/02/17 08:12:41 UTC
[GitHub] sijie closed pull request #1087: ISSUE #1075: Add a noop digest implentation
sijie closed pull request #1087: ISSUE #1075: Add a noop digest implentation
URL: https://github.com/apache/bookkeeper/pull/1087
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-proto/src/main/proto/DataFormats.proto b/bookkeeper-proto/src/main/proto/DataFormats.proto
index 5a5d65f79..823257c59 100644
--- a/bookkeeper-proto/src/main/proto/DataFormats.proto
+++ b/bookkeeper-proto/src/main/proto/DataFormats.proto
@@ -46,6 +46,7 @@ message LedgerMetadataFormat {
CRC32 = 1;
HMAC = 2;
CRC32C = 3;
+ DUMMY = 4;
}
optional DigestType digestType = 7;
optional bytes password = 8;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
index 79dd6254e..fb315de1b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
@@ -658,9 +658,12 @@ public RegistrationClient getRegClient() {
* The CRC32C, which use SSE processor instruction, has better performance than CRC32.
* Legacy DigestType for backward compatibility. If we want to add new DigestType,
* we should add it in here, client.api.DigestType and DigestType in DataFormats.proto.
+ * If the digest type is set/passed in as DUMMY, a dummy digest is added/checked.
+ * This DUMMY digest is mostly for test purposes or in situations/use-cases
+ * where digest is considered a overhead.
*/
public enum DigestType {
- MAC, CRC32, CRC32C;
+ MAC, CRC32, CRC32C, DUMMY;
public static DigestType fromApiDigestType(org.apache.bookkeeper.client.api.DigestType digestType) {
switch (digestType) {
@@ -670,6 +673,8 @@ public static DigestType fromApiDigestType(org.apache.bookkeeper.client.api.Dige
return DigestType.CRC32;
case CRC32C:
return DigestType.CRC32C;
+ case DUMMY:
+ return DigestType.DUMMY;
default:
throw new IllegalArgumentException("Unable to convert digest type " + digestType);
}
@@ -682,6 +687,8 @@ public static DigestType fromApiDigestType(org.apache.bookkeeper.client.api.Dige
return DataFormats.LedgerMetadataFormat.DigestType.CRC32;
case CRC32C:
return DataFormats.LedgerMetadataFormat.DigestType.CRC32C;
+ case DUMMY:
+ return DataFormats.LedgerMetadataFormat.DigestType.DUMMY;
default:
throw new IllegalArgumentException("Unable to convert digest type " + digestType);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index d37e7fc80..d58d8c66b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -237,6 +237,8 @@ public DigestType getDigestType() {
return DigestType.CRC32;
case CRC32C:
return DigestType.CRC32C;
+ case DUMMY:
+ return DigestType.DUMMY;
default:
throw new IllegalArgumentException("Unable to convert digest type " + digestType);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
index 728a24691..14c8f1b81 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
@@ -43,5 +43,9 @@
/**
* Entries are verified by applied CRC32C algorithm.
*/
- CRC32C
+ CRC32C,
+ /**
+ * Entries are not verified.
+ */
+ DUMMY,
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
index 67386a6d4..cf6b315b5 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
@@ -72,6 +72,8 @@ public static DigestManager instantiate(long ledgerId, byte[] passwd, DigestType
return new CRC32DigestManager(ledgerId);
case CRC32C:
return new CRC32CDigestManager(ledgerId);
+ case DUMMY:
+ return new DummyDigestManager(ledgerId);
default:
throw new GeneralSecurityException("Unknown checksum type: " + digestType);
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DummyDigestManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DummyDigestManager.java
new file mode 100644
index 000000000..aedc5ad67
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DummyDigestManager.java
@@ -0,0 +1,44 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.proto.checksum;
+
+import io.netty.buffer.ByteBuf;
+
+
+/**
+ * This class provides a noop digest implementation.
+ */
+public class DummyDigestManager extends DigestManager {
+ public DummyDigestManager(long ledgerId) {
+ super(ledgerId);
+ }
+
+ @Override
+ int getMacCodeLength() {
+ return 0;
+ }
+
+ @Override
+ void update(ByteBuf buffer) {}
+
+ @Override
+ void populateValueAndReset(ByteBuf buffer) {}
+}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
index bae15b81a..5135d8e9c 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
@@ -149,6 +149,34 @@ public void testWithMultipleBookieFailuresInLastEnsemble() throws Exception {
lh.close();
}
+ /**
+ * Verify the functionality Ledgers with different digests.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testLedgerDigestTest() throws Exception {
+ for (DigestType type: DigestType.values()) {
+ lh = bkc.createLedger(5, 3, 2, type, ledgerPassword);
+
+ for (int i = 0; i < numEntriesToWrite; i++) {
+ ByteBuffer entry = ByteBuffer.allocate(4);
+ entry.putInt(rng.nextInt(maxInt));
+ entry.position(0);
+
+ entries1.add(entry.array());
+ lh.addEntry(entry.array());
+ }
+
+ readEntries(lh, entries1);
+
+ long lid = lh.getId();
+ lh.close();
+ bkc.deleteLedger(lid);
+ entries1.clear();
+ }
+ }
+
/**
* Verify the functionality of Advanced Ledger which returns
* LedgerHandleAdv. LedgerHandleAdv takes entryId for addEntry, and let
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
index 1ebfb8fa8..68ad40912 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/MockBookKeeperTestCase.java
@@ -50,7 +50,6 @@
import org.apache.bookkeeper.proto.BookieClient;
import org.apache.bookkeeper.proto.BookieProtocol;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks;
-import org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.DigestType;
import org.apache.bookkeeper.proto.checksum.DigestManager;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.util.OrderedSafeExecutor;
@@ -162,6 +161,16 @@ protected NullStatsLogger setupLoggers() {
return nullStatsLogger;
}
+ private DigestManager getDigestType(long ledgerId) throws GeneralSecurityException {
+ LedgerMetadata metadata = mockLedgerMetadataRegistry.get(ledgerId);
+ return DigestManager.instantiate(
+ ledgerId,
+ metadata.getPassword(),
+ org.apache.bookkeeper.client.BookKeeper.DigestType.toProtoDigestType(
+ org.apache.bookkeeper.client.BookKeeper.DigestType.fromApiDigestType(
+ metadata.getDigestType())));
+ }
+
@After
public void tearDown() {
scheduler.shutdown();
@@ -338,7 +347,7 @@ protected void setupBookieClientReadEntry() {
executor.submitOrdered(ledgerId, () -> {
DigestManager macManager = null;
try {
- macManager = DigestManager.instantiate(ledgerId, new byte[2], DigestType.CRC32);
+ macManager = getDigestType(ledgerId);
} catch (GeneralSecurityException gse){
LOG.error("Initialize macManager fail", gse);
}
@@ -373,7 +382,7 @@ protected void setupBookieClientReadEntry() {
executor.submitOrdered(ledgerId, () -> {
DigestManager macManager = null;
try {
- macManager = DigestManager.instantiate(ledgerId, new byte[2], DigestType.CRC32);
+ macManager = getDigestType(ledgerId);
} catch (GeneralSecurityException gse){
LOG.error("Initialize macManager fail", gse);
}
@@ -396,13 +405,13 @@ protected void setupBookieClientReadEntry() {
any(BookkeeperInternalCallbacks.ReadEntryCallback.class), any());
}
- private static byte[] extractEntryPayload(long ledgerId, long entryId, ByteBuf toSend)
+ private byte[] extractEntryPayload(long ledgerId, long entryId, ByteBuf toSend)
throws BKException.BKDigestMatchException {
ByteBuf toSendCopy = Unpooled.copiedBuffer(toSend);
toSendCopy.resetReaderIndex();
DigestManager macManager = null;
try {
- macManager = DigestManager.instantiate(ledgerId, new byte[2], DigestType.CRC32);
+ macManager = getDigestType(ledgerId);
} catch (GeneralSecurityException gse){
LOG.error("Initialize macManager fail", gse);
}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperApiTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperApiTest.java
index 9a3712223..fd30f820a 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperApiTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperApiTest.java
@@ -53,6 +53,7 @@
*/
public class BookKeeperApiTest extends MockBookKeeperTestCase {
+ private static final byte[] bigData = new byte[1024];
private static final byte[] data = "foo".getBytes(UTF_8);
private static final byte[] password = "password".getBytes(UTF_8);
@@ -162,6 +163,40 @@ public void testOpenLedgerUnauthorized() throws Exception {
}
}
+ /**
+ * Verify the functionality Ledgers with different digests.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testLedgerDigests() throws Exception {
+ for (DigestType type: DigestType.values()) {
+ long lId;
+ try (WriteHandle writer = result(newCreateLedgerOp()
+ .withAckQuorumSize(1)
+ .withWriteQuorumSize(2)
+ .withEnsembleSize(3)
+ .withDigestType(type)
+ .withPassword(password)
+ .execute())) {
+ lId = writer.getId();
+ assertEquals(-1L, writer.getLastAddPushed());
+ result(writer.append(ByteBuffer.wrap(bigData)));
+ assertEquals(bigData.length, writer.getLength());
+ }
+ try (ReadHandle reader = result(newOpenLedgerOp()
+ .withDigestType(type)
+ .withPassword(password)
+ .withLedgerId(lId)
+ .execute())) {
+ LedgerEntries entries = result(reader.read(0, 0));
+ checkEntries(entries, bigData);
+ }
+ result(newDeleteLedgerOp().withLedgerId(lId).execute());
+ }
+ }
+
+
@Test(expected = BKDigestMatchException.class)
public void testOpenLedgerDigestUnmatched() throws Exception {
long lId;
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services