You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2019/08/08 22:40:27 UTC
[hadoop] branch trunk updated: HDDS-1863. Freon RandomKeyGenerator
even if keySize is set to 0, it returns some random data to key. (#1167)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new aa5f445 HDDS-1863. Freon RandomKeyGenerator even if keySize is set to 0, it returns some random data to key. (#1167)
aa5f445 is described below
commit aa5f445fb9d06f9967aadf305fa3cd509a16b982
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Thu Aug 8 15:40:19 2019 -0700
HDDS-1863. Freon RandomKeyGenerator even if keySize is set to 0, it returns some random data to key. (#1167)
---
.../apache/hadoop/ozone/freon/RandomKeyGenerator.java | 10 ++--------
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 19 +++++++++++++++++++
2 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 5198ac3..7cfd1fe 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -26,7 +26,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
@@ -263,9 +262,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
// Compute the common initial digest for all keys without their UUID
if (validateWrites) {
commonInitialMD = DigestUtils.getDigest(DIGEST_ALGORITHM);
- int uuidLength = UUID.randomUUID().toString().length();
- keySize = Math.max(uuidLength, keySize);
- for (long nrRemaining = keySize - uuidLength; nrRemaining > 0;
+ for (long nrRemaining = keySize; nrRemaining > 0;
nrRemaining -= bufferSize) {
int curSize = (int)Math.min(bufferSize, nrRemaining);
commonInitialMD.update(keyValueBuffer, 0, curSize);
@@ -682,7 +679,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
+ RandomStringUtils.randomNumeric(5);
LOG.trace("Adding key: {} in bucket: {} of volume: {}",
keyName, bucketName, volumeName);
- byte[] randomValue = DFSUtil.string2Bytes(UUID.randomUUID().toString());
try {
try (Scope scope = GlobalTracer.get().buildSpan("createKey")
.startActive(true)) {
@@ -697,12 +693,11 @@ public final class RandomKeyGenerator implements Callable<Void> {
try (Scope writeScope = GlobalTracer.get().buildSpan("writeKeyData")
.startActive(true)) {
long keyWriteStart = System.nanoTime();
- for (long nrRemaining = keySize - randomValue.length;
+ for (long nrRemaining = keySize;
nrRemaining > 0; nrRemaining -= bufferSize) {
int curSize = (int) Math.min(bufferSize, nrRemaining);
os.write(keyValueBuffer, 0, curSize);
}
- os.write(randomValue);
os.close();
long keyWriteDuration = System.nanoTime() - keyWriteStart;
@@ -716,7 +711,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
if (validateWrites) {
MessageDigest tmpMD = (MessageDigest) commonInitialMD.clone();
- tmpMD.update(randomValue);
boolean validate = validationQueue.offer(
new KeyValidate(bucket, keyName, tmpMD.digest()));
if (validate) {
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
index 748972e..45ea23d 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
@@ -129,6 +129,25 @@ public class TestRandomKeyGenerator {
}
@Test
+ public void fileWithSizeZero() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(1);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(1);
+ randomKeyGenerator.setNumOfThreads(1);
+ randomKeyGenerator.setKeySize(0);
+ randomKeyGenerator.setFactor(ReplicationFactor.THREE);
+ randomKeyGenerator.setType(ReplicationType.RATIS);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(1, randomKeyGenerator.getSuccessfulValidationCount());
+ }
+
+ @Test
public void testThreadPoolSize() throws Exception {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org