You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2017/07/27 04:41:08 UTC
hadoop git commit: HDFS-12145. Ozone: OzoneFileSystem: Ozone & KSM
should support / delimited key names. Contributed by Mukul Kumar Singh.
Repository: hadoop
Updated Branches:
refs/heads/DFS-7240 [created] fe0a957e9
HDFS-12145. Ozone: OzoneFileSystem: Ozone & KSM should support / delimited key names. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe0a957e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe0a957e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe0a957e
Branch: refs/heads/DFS-7240
Commit: fe0a957e92d506330ba005abba18f665eb5ac1cc
Parents: f6ed186
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Jul 27 12:40:40 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Jul 27 12:40:40 2017 +0800
----------------------------------------------------------------------
.../hadoop/scm/storage/ChunkOutputStream.java | 4 +-
.../hadoop/ozone/web/interfaces/Keys.java | 2 +-
.../hadoop/ozone/web/client/TestKeys.java | 56 ++++++++++++++++++--
.../hadoop/ozone/web/client/TestKeysRatis.java | 20 +++++++
4 files changed, 76 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe0a957e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
index d93d2cf..8daded5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
@@ -28,6 +28,7 @@ import java.util.UUID;
import com.google.protobuf.ByteString;
+import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
@@ -209,7 +210,8 @@ public class ChunkOutputStream extends OutputStream {
ChunkInfo chunk = ChunkInfo
.newBuilder()
.setChunkName(
- key + "_stream_" + streamId + "_chunk_" + ++chunkIndex)
+ DigestUtils.md5Hex(key) + "_stream_"
+ + streamId + "_chunk_" + ++chunkIndex)
.setOffset(0)
.setLen(data.size())
.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe0a957e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
index b6885a7..e64c8ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
@@ -40,7 +40,7 @@ import java.io.InputStream;
* This interface defines operations permitted on a key.
*/
-@Path("/{volume}/{bucket}/{keys}")
+@Path("/{volume}/{bucket}/{keys:.*}")
public interface Keys {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe0a957e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 516c571..d9168ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.web.client;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.math.RandomUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -28,6 +30,7 @@ import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Stat
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
@@ -122,17 +125,41 @@ public class TestKeys {
return tmpFile;
}
+ /**
+ * This function generates multi part key which are delimited by a certain
+ * delimiter. Different parts of key are random string of random length
+ * between 0 - 4. Number of parts of the keys are between 0 and 5.
+ *
+ * @param delimiter delimiter used to delimit parts of string
+ * @return Key composed of multiple parts delimited by "/"
+ */
+ static String getMultiPartKey(String delimiter) {
+ int numParts = RandomUtils.nextInt(5) + 1;
+ String[] nameParts = new String[numParts];
+ for (int i = 0; i < numParts; i++) {
+ int stringLength = numParts == 1 ? 5 : RandomUtils.nextInt(5);
+ nameParts[i] = RandomStringUtils.randomAlphanumeric(stringLength);
+ }
+ return StringUtils.join(delimiter, nameParts);
+ }
+
static class PutHelper {
private final OzoneRestClient client;
private final String dir;
+ private final String keyName;
OzoneVolume vol;
OzoneBucket bucket;
File file;
PutHelper(OzoneRestClient client, String dir) {
+ this(client, dir, OzoneUtils.getRequestID().toLowerCase());
+ }
+
+ PutHelper(OzoneRestClient client, String dir, String key) {
this.client = client;
this.dir = dir;
+ this.keyName = key;
}
public OzoneVolume getVol() {
@@ -163,8 +190,8 @@ public class TestKeys {
String bucketName = OzoneUtils.getRequestID().toLowerCase();
bucket = vol.createBucket(bucketName, acls, StorageType.DEFAULT);
- String keyName = OzoneUtils.getRequestID().toLowerCase();
- file = createRandomDataFile(dir, keyName, 1024);
+ String fileName = OzoneUtils.getRequestID().toLowerCase();
+ file = createRandomDataFile(dir, fileName, 1024);
bucket.putKey(keyName, file);
return keyName;
@@ -174,7 +201,12 @@ public class TestKeys {
@Test
public void testPutKey() throws OzoneException {
+ // Test non-delimited keys
runTestPutKey(new PutHelper(ozoneRestClient, path));
+ // Test key delimited by a random delimiter
+ String delimiter = RandomStringUtils.randomAscii(1);
+ runTestPutKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
static void runTestPutKey(PutHelper helper) throws OzoneException {
@@ -229,6 +261,10 @@ public class TestKeys {
throws OzoneException, IOException, URISyntaxException {
runTestPutAndGetKeyWithDnRestart(
new PutHelper(ozoneRestClient, path), ozoneCluster);
+ String delimiter = RandomStringUtils.randomAscii(1);
+ runTestPutAndGetKeyWithDnRestart(
+ new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)), ozoneCluster);
}
static void runTestPutAndGetKeyWithDnRestart(
@@ -263,6 +299,9 @@ public class TestKeys {
@Test
public void testPutAndGetKey() throws OzoneException, IOException {
runTestPutAndGetKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAscii(1);
+ runTestPutAndGetKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
static void runTestPutAndGetKey(PutHelper helper)
@@ -326,6 +365,9 @@ public class TestKeys {
@Test
public void testPutAndDeleteKey() throws OzoneException, IOException {
runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAscii(1);
+ runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
static void runTestPutAndDeleteKey(PutHelper helper)
@@ -347,6 +389,9 @@ public class TestKeys {
@Test
public void testPutAndListKey() throws OzoneException, IOException {
runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAscii(1);
+ runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
static void runTestPutAndListKey(PutHelper helper)
@@ -378,9 +423,9 @@ public class TestKeys {
Assert.assertEquals(keyList2.size(), 1);
// test startKey parameter of list keys
- keyList1 = helper.getBucket().listKeys("100", "list-key4", null);
+ keyList1 = helper.getBucket().listKeys("100", "list-key4", "list-key");
keyList2 = client.listKeys(helper.getVol().getVolumeName(),
- helper.getBucket().getBucketName(), "100", "list-key4", null);
+ helper.getBucket().getBucketName(), "100", "list-key4", "list-key");
Assert.assertEquals(keyList1.size(), 5);
Assert.assertEquals(keyList2.size(), 5);
@@ -416,6 +461,9 @@ public class TestKeys {
@Test
public void testGetKeyInfo() throws OzoneException, IOException {
runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAscii(1);
+ runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
static void runTestGetKeyInfo(PutHelper helper) throws OzoneException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe0a957e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
index b0fea90..1d33644 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.web.client;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@@ -57,6 +58,9 @@ public class TestKeysRatis {
@Test
public void testPutKey() throws OzoneException {
runTestPutKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAlphanumeric(1);
+ runTestPutKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
@Test
@@ -64,25 +68,41 @@ public class TestKeysRatis {
throws OzoneException, IOException, URISyntaxException {
runTestPutAndGetKeyWithDnRestart(
new PutHelper(ozoneRestClient, path), suite.getCluster());
+ String delimiter = RandomStringUtils.randomAlphanumeric(1);
+ runTestPutAndGetKeyWithDnRestart(
+ new PutHelper(ozoneRestClient, path, getMultiPartKey(delimiter)),
+ suite.getCluster());
}
@Test
public void testPutAndGetKey() throws OzoneException, IOException {
runTestPutAndGetKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAlphanumeric(1);
+ runTestPutAndGetKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
@Test
public void testPutAndDeleteKey() throws OzoneException, IOException {
runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAlphanumeric(1);
+ runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
@Test
public void testPutAndListKey() throws OzoneException, IOException {
runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAlphanumeric(1);
+ runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
@Test
public void testGetKeyInfo() throws OzoneException, IOException {
runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
+ String delimiter = RandomStringUtils.randomAlphanumeric(1);
+ runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,
+ getMultiPartKey(delimiter)));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org