You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2017/07/27 17:40:00 UTC
hadoop git commit: HDFS-12059. Ozone: OzoneClient: OzoneClientImpl
Add setBucketProperty and delete calls. Contributed by Nandakumar.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 5b3cac765 -> 5e4707603
HDFS-12059. Ozone: OzoneClient: OzoneClientImpl Add setBucketProperty and delete calls. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e470760
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e470760
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e470760
Branch: refs/heads/HDFS-7240
Commit: 5e47076032eb18c0371088ab15a7c9f334203893
Parents: 5b3cac7
Author: Chen Liang <cl...@apache.org>
Authored: Thu Jul 27 10:39:33 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Thu Jul 27 10:39:33 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneClient.java | 2 +-
.../apache/hadoop/ozone/OzoneClientImpl.java | 116 ++++++++++++++-
.../hadoop/ozone/TestOzoneClientImpl.java | 147 ++++++++++++++++++-
3 files changed, 254 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e470760/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java
index 241716f..dd52a57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java
@@ -403,7 +403,7 @@ public interface OzoneClient {
*
* @throws IOException
*/
- OzoneKey getkeyDetails(String volumeName, String bucketName,
+ OzoneKey getKeyDetails(String volumeName, String bucketName,
String keyName)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e470760/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java
index 01807b0..3c79778 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java
@@ -20,9 +20,16 @@ package org.apache.hadoop.ozone;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.ozone.protocol.proto
+ .ContainerProtos.ChunkInfo;
+import org.apache.hadoop.hdfs.ozone.protocol.proto
+ .ContainerProtos.GetKeyResponseProto;
+import org.apache.hadoop.hdfs.ozone.protocol.proto
+ .ContainerProtos.KeyData;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ksm.helpers.KsmKeyInfo;
@@ -45,6 +52,7 @@ import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.scm.storage.ChunkInputStream;
import org.apache.hadoop.scm.storage.ChunkOutputStream;
import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.security.UserGroupInformation;
@@ -375,34 +383,65 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
public void addBucketAcls(String volumeName, String bucketName,
List<OzoneAcl> addAcls)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(addAcls);
+ KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
+ builder.setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setAddAcls(addAcls);
+ keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void removeBucketAcls(String volumeName, String bucketName,
List<OzoneAcl> removeAcls)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(removeAcls);
+ KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
+ builder.setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setRemoveAcls(removeAcls);
+ keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void setBucketVersioning(String volumeName, String bucketName,
Versioning versioning)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(versioning);
+ KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
+ builder.setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setIsVersionEnabled(getBucketVersioningFlag(
+ versioning));
+ keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void setBucketStorageType(String volumeName, String bucketName,
StorageType storageType)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(storageType);
+ KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
+ builder.setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setStorageType(storageType);
+ keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void deleteBucket(String volumeName, String bucketName)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ keySpaceManagerClient.deleteBucket(volumeName, bucketName);
}
@Override
@@ -480,14 +519,54 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
public OzoneInputStream getKey(String volumeName, String bucketName,
String keyName)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(keyName);
+ String requestId = UUID.randomUUID().toString();
+ KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .build();
+ KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
+ String containerKey = buildContainerKey(volumeName,
+ bucketName, keyName);
+ String containerName = keyInfo.getContainerName();
+ XceiverClientSpi xceiverClient = getContainer(containerName);
+ boolean success = false;
+ try {
+ LOG.debug("get key accessing {} {}",
+ xceiverClient.getPipeline().getContainerName(), containerKey);
+ KeyData containerKeyData = KeyData.newBuilder().setContainerName(
+ xceiverClient.getPipeline().getContainerName())
+ .setName(containerKey).build();
+ GetKeyResponseProto response = ContainerProtocolCalls
+ .getKey(xceiverClient, containerKeyData, requestId);
+ List<ChunkInfo> chunks = response.getKeyData().getChunksList();
+ success = true;
+ return new OzoneInputStream(new ChunkInputStream(
+ containerKey, xceiverClientManager, xceiverClient,
+ chunks, requestId));
+ } finally {
+ if (!success) {
+ xceiverClientManager.releaseClient(xceiverClient);
+ }
+ }
}
@Override
public void deleteKey(String volumeName, String bucketName,
String keyName)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(keyName);
+ KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .build();
+ keySpaceManagerClient.deleteKey(keyArgs);
}
@Override
@@ -498,7 +577,7 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
}
@Override
- public OzoneKey getkeyDetails(String volumeName, String bucketName,
+ public OzoneKey getKeyDetails(String volumeName, String bucketName,
String keyName)
throws IOException {
Preconditions.checkNotNull(volumeName);
@@ -514,6 +593,27 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
return new OzoneKey(keyInfo);
}
+ /**
+ * Converts Versioning to boolean.
+ *
+ * @param version
+ * @return corresponding boolean value
+ */
+ private boolean getBucketVersioningFlag(
+ Versioning version) {
+ if(version != null) {
+ switch(version) {
+ case ENABLED:
+ return true;
+ case DISABLED:
+ case NOT_DEFINED:
+ default:
+ return false;
+ }
+ }
+ return false;
+ }
+
@Override
public void close() throws IOException {
if(xceiverClientManager != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e470760/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java
index 30ef144..b861f7d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java
@@ -19,14 +19,19 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.ozone.io.OzoneInputStream;
import org.apache.hadoop.ozone.io.OzoneOutputStream;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.UUID;
/**
@@ -34,6 +39,9 @@ import java.util.UUID;
*/
public class TestOzoneClientImpl {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
private static MiniOzoneCluster cluster = null;
private static OzoneClientImpl ozClient = null;
@@ -101,6 +109,38 @@ public class TestOzoneClientImpl {
}
@Test
+ public void testSetVolumeOwner()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ ozClient.setVolumeOwner(volumeName, "test");
+ OzoneVolume volume = ozClient.getVolumeDetails(volumeName);
+ Assert.assertEquals("test", volume.getOwnerName());
+ }
+
+ @Test
+ public void testSetVolumeQuota()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ ozClient.setVolumeQuota(volumeName, 10000000000L);
+ OzoneVolume volume = ozClient.getVolumeDetails(volumeName);
+ Assert.assertEquals(10000000000L, volume.getQuota());
+ }
+
+ @Test
+ public void testDeleteVolume()
+ throws IOException, OzoneException {
+ thrown.expectMessage("Info Volume failed, error");
+ String volumeName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ OzoneVolume volume = ozClient.getVolumeDetails(volumeName);
+ Assert.assertNotNull(volume);
+ ozClient.deleteVolume(volumeName);
+ ozClient.getVolumeDetails(volumeName);
+ }
+
+ @Test
public void testCreateBucket()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
@@ -185,6 +225,85 @@ public class TestOzoneClientImpl {
}
@Test
+ public void testAddBucketAcl()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ ozClient.createBucket(volumeName, bucketName);
+ List<OzoneAcl> acls = new ArrayList<>();
+ acls.add(new OzoneAcl(
+ OzoneAcl.OzoneACLType.USER, "test",
+ OzoneAcl.OzoneACLRights.READ_WRITE));
+ ozClient.addBucketAcls(volumeName, bucketName, acls);
+ OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
+ Assert.assertEquals(bucketName, bucket.getBucketName());
+ Assert.assertTrue(bucket.getAcls().contains(acls.get(0)));
+ }
+
+ @Test
+ public void testRemoveBucketAcl()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test",
+ OzoneAcl.OzoneACLRights.READ_WRITE);
+ ozClient.createVolume(volumeName);
+ ozClient.createBucket(volumeName, bucketName, userAcl);
+ List<OzoneAcl> acls = new ArrayList<>();
+ acls.add(userAcl);
+ ozClient.removeBucketAcls(volumeName, bucketName, acls);
+ OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
+ Assert.assertEquals(bucketName, bucket.getBucketName());
+ Assert.assertTrue(!bucket.getAcls().contains(acls.get(0)));
+ }
+
+ @Test
+ public void testSetBucketVersioning()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ ozClient.createBucket(volumeName, bucketName);
+ ozClient.setBucketVersioning(volumeName, bucketName,
+ OzoneConsts.Versioning.ENABLED);
+ OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
+ Assert.assertEquals(bucketName, bucket.getBucketName());
+ Assert.assertEquals(OzoneConsts.Versioning.ENABLED,
+ bucket.getVersioning());
+ }
+
+ @Test
+ public void testSetBucketStorageType()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ ozClient.createBucket(volumeName, bucketName);
+ ozClient.setBucketStorageType(volumeName, bucketName,
+ StorageType.SSD);
+ OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
+ Assert.assertEquals(bucketName, bucket.getBucketName());
+ Assert.assertEquals(StorageType.SSD, bucket.getStorageType());
+ }
+
+
+ @Test
+ public void testDeleteBucket()
+ throws IOException, OzoneException {
+ thrown.expectMessage("Info Bucket failed, error");
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ ozClient.createVolume(volumeName);
+ ozClient.createBucket(volumeName, bucketName);
+ OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
+ Assert.assertNotNull(bucket);
+ ozClient.deleteBucket(volumeName, bucketName);
+ ozClient.getBucketDetails(volumeName, bucketName);
+ }
+
+
+ @Test
public void testPutKey()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
@@ -196,9 +315,33 @@ public class TestOzoneClientImpl {
OzoneOutputStream out = ozClient.createKey(volumeName, bucketName,
keyName, value.getBytes().length);
out.write(value.getBytes());
- OzoneKey key = ozClient.getkeyDetails(volumeName, bucketName, keyName);
+ out.close();
+ OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName);
+ Assert.assertEquals(keyName, key.getKeyName());
+ OzoneInputStream is = ozClient.getKey(volumeName, bucketName, keyName);
+ byte[] fileContent = new byte[value.getBytes().length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent));
+ }
+
+ @Test
+ public void testDeleteKey()
+ throws IOException, OzoneException {
+ thrown.expectMessage("Lookup key failed, error");
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+ String value = "sample value";
+ ozClient.createVolume(volumeName);
+ ozClient.createBucket(volumeName, bucketName);
+ OzoneOutputStream out = ozClient.createKey(volumeName, bucketName,
+ keyName, value.getBytes().length);
+ out.write(value.getBytes());
+ out.close();
+ OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName);
Assert.assertEquals(keyName, key.getKeyName());
- //Content validation has to be done after getKey implementation.
+ ozClient.deleteKey(volumeName, bucketName, keyName);
+ ozClient.getKeyDetails(volumeName, bucketName, keyName);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org