You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2019/04/08 23:56:03 UTC
[hadoop] branch HDDS-1393 created (now 8f65739)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a change to branch HDDS-1393
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
at 8f65739 fix review comments
This branch includes the following new commits:
new 1fdefcb HDDS-1393. Convert all OM Bucket related operations to HA model.
new 8f65739 fix review comments
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDDS-1393. Convert all OM Bucket related operations
to HA model.
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch HDDS-1393
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1fdefcb4079d5aade8bc4e79ef100e2e8912e481
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Fri Apr 5 15:25:39 2019 -0700
HDDS-1393. Convert all OM Bucket related operations to HA model.
---
.../ozone/om/protocol/OzoneManagerHAProtocol.java | 52 +++++++++
.../src/main/proto/OzoneManagerProtocol.proto | 1 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 6 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 69 ++++++++++--
.../org/apache/hadoop/ozone/om/BucketManager.java | 29 ++++-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 92 ++++++++++++++--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 65 ++++++++++-
.../hadoop/ozone/om/S3BucketManagerImpl.java | 15 ++-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 3 +
.../OzoneManagerHARequestHandlerImpl.java | 122 +++++++++++++++++++++
10 files changed, 430 insertions(+), 24 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java
index ad2bc31..eb514d0 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.om.protocol;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDeleteVolumeResponse;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -166,4 +168,54 @@ public interface OzoneManagerHAProtocol {
*/
void applyDeleteVolume(String volume, String owner,
VolumeList newVolumeList) throws IOException;
+
+ /**
+ * Start Create Bucket Transaction.
+ * @param omBucketInfo
+ * @return OmBucketInfo
+ * @throws IOException
+ */
+ OmBucketInfo startCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+
+ /**
+ * Apply Create Bucket Changes to OM DB.
+ * @param omBucketInfo
+ * @throws IOException
+ */
+ void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+
+ /**
+ * Start Delete Bucket Transaction.
+ * @param volumeName
+ * @param bucketName
+ * @throws IOException
+ */
+ void startDeleteBucket(String volumeName, String bucketName)
+ throws IOException;
+
+ /**
+ * Apply Delete Bucket changes to OM DB.
+ * @param volumeName
+ * @param bucketName
+ * @throws IOException
+ */
+ void applyDeleteBucket(String volumeName, String bucketName)
+ throws IOException;
+
+ /**
+ * Start SetBucket Property Transaction.
+ * @param omBucketArgs
+ * @return OmBucketInfo
+ * @throws IOException
+ */
+ OmBucketInfo startSetBucketProperty(OmBucketArgs omBucketArgs)
+ throws IOException;
+
+ /**
+ * Apply SetBucket Property changes to OM DB.
+ * @param omBucketInfo
+ * @throws IOException
+ */
+ void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
+
}
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 4536e87..e641464 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -479,6 +479,7 @@ message InfoBucketResponse {
message SetBucketPropertyRequest {
required BucketArgs bucketArgs = 1;
+ optional BucketInfo bucketInfo = 2;
}
message SetBucketPropertyResponse {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index 5d739c2..39fada8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -171,11 +171,11 @@ public class TestOmMetrics {
Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
- Mockito.doNothing().when(mockBm).createBucket(null);
- Mockito.doNothing().when(mockBm).createBucket(null);
+ Mockito.doReturn(null).when(mockBm).createBucket(null);
+ Mockito.doReturn(null).when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).deleteBucket(null, null);
Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
- Mockito.doNothing().when(mockBm).setBucketProperty(null);
+ Mockito.doReturn(null).when(mockBm).setBucketProperty(null);
Mockito.doReturn(null).when(mockBm).listBuckets(null, null, null, 0);
HddsWhiteboxTestUtils.setInternalState(
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
index 5f62af1..422a02c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
@@ -20,11 +20,13 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.LogVerificationAppender;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.OzoneTestUtils;
+import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -38,6 +40,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.util.Time;
import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.Assert;
@@ -125,23 +128,32 @@ public class TestOzoneManagerHA {
}
}
- @Test
- public void testAllVolumeOperations() throws Exception {
+
+ private OzoneVolume createAndCheckVolume(String volumeName)
+ throws Exception {
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-
VolumeArgs createVolumeArgs = VolumeArgs.newBuilder()
.setOwner(userName)
.setAdmin(adminName)
.build();
objectStore.createVolume(volumeName, createVolumeArgs);
- OzoneVolume retVolumeinfo = objectStore.getVolume(volumeName);
- Assert.assertTrue(retVolumeinfo.getName().equals(volumeName));
- Assert.assertTrue(retVolumeinfo.getOwner().equals(userName));
- Assert.assertTrue(retVolumeinfo.getAdmin().equals(adminName));
+ OzoneVolume retVolume = objectStore.getVolume(volumeName);
+
+ Assert.assertTrue(retVolume.getName().equals(volumeName));
+ Assert.assertTrue(retVolume.getOwner().equals(userName));
+ Assert.assertTrue(retVolume.getAdmin().equals(adminName));
+
+ return retVolume;
+ }
+ @Test
+ public void testAllVolumeOperations() throws Exception {
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+
+ createAndCheckVolume(volumeName);
objectStore.deleteVolume(volumeName);
@@ -152,6 +164,47 @@ public class TestOzoneManagerHA {
() -> objectStore.deleteVolume(volumeName));
}
+
+ @Test
+ public void testAllBucketOperations() throws Exception {
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "volume" + RandomStringUtils.randomNumeric(5);
+
+ OzoneVolume retVolume = createAndCheckVolume(volumeName);
+
+ BucketArgs bucketArgs =
+ BucketArgs.newBuilder().setStorageType(StorageType.DISK)
+ .setVersioning(true).build();
+
+
+ retVolume.createBucket(bucketName, bucketArgs);
+
+
+ OzoneBucket ozoneBucket = retVolume.getBucket(bucketName);
+
+ Assert.assertEquals(volumeName, ozoneBucket.getVolumeName());
+ Assert.assertEquals(bucketName, ozoneBucket.getName());
+ Assert.assertTrue(ozoneBucket.getVersioning());
+ Assert.assertEquals(StorageType.DISK, ozoneBucket.getStorageType());
+ Assert.assertTrue(ozoneBucket.getCreationTime() <= Time.now());
+
+
+ // Change versioning to false
+ ozoneBucket.setVersioning(false);
+
+ ozoneBucket = retVolume.getBucket(bucketName);
+ Assert.assertFalse(ozoneBucket.getVersioning());
+
+ retVolume.deleteBucket(bucketName);
+
+ OzoneTestUtils.expectOmException(OMException.ResultCodes.BUCKET_NOT_FOUND,
+ () -> retVolume.deleteBucket(bucketName));
+
+
+
+ }
+
/**
* Test a client request when all OM nodes are running. The request should
* succeed.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java
index e82de80..460ac11 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java
@@ -30,7 +30,16 @@ public interface BucketManager {
* Creates a bucket.
* @param bucketInfo - OmBucketInfo for creating bucket.
*/
- void createBucket(OmBucketInfo bucketInfo) throws IOException;
+ OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException;
+
+ /**
+ * Apply Create Bucket changes to OM DB.
+ * @param omBucketInfo
+ * @throws IOException
+ */
+ void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+
+
/**
* Returns Bucket Information.
* @param volumeName - Name of the Volume.
@@ -44,7 +53,14 @@ public interface BucketManager {
* @param args - BucketArgs.
* @throws IOException
*/
- void setBucketProperty(OmBucketArgs args) throws IOException;
+ OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException;
+
+ /**
+ * Apply SetBucket Property changes to OM DB.
+ * @param omBucketInfo
+ * @throws IOException
+ */
+ void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
/**
* Deletes an existing empty bucket from volume.
@@ -55,6 +71,15 @@ public interface BucketManager {
void deleteBucket(String volumeName, String bucketName) throws IOException;
/**
+ * Apply Delete Bucket changes to OM DB.
+ * @param volumeName
+ * @param bucketName
+ * @throws IOException
+ */
+ void applyDeleteBucket(String volumeName, String bucketName)
+ throws IOException;
+
+ /**
* Returns a list of buckets represented by {@link OmBucketInfo}
* in the given volume.
*
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
index b15e692..68cd2a9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
@@ -50,19 +50,27 @@ public class BucketManagerImpl implements BucketManager {
private final OMMetadataManager metadataManager;
private final KeyProviderCryptoExtension kmsProvider;
+ private final boolean isRatisEnabled;
+
/**
* Constructs BucketManager.
*
* @param metadataManager
*/
public BucketManagerImpl(OMMetadataManager metadataManager) {
- this(metadataManager, null);
+ this(metadataManager, null, false);
}
public BucketManagerImpl(OMMetadataManager metadataManager,
KeyProviderCryptoExtension kmsProvider) {
+ this(metadataManager, kmsProvider, false);
+ }
+
+ public BucketManagerImpl(OMMetadataManager metadataManager,
+ KeyProviderCryptoExtension kmsProvider, boolean isRatisEnabled) {
this.metadataManager = metadataManager;
this.kmsProvider = kmsProvider;
+ this.isRatisEnabled = isRatisEnabled;
}
KeyProviderCryptoExtension getKMSProvider() {
@@ -94,7 +102,7 @@ public class BucketManagerImpl implements BucketManager {
* @param bucketInfo - OmBucketInfo.
*/
@Override
- public void createBucket(OmBucketInfo bucketInfo) throws IOException {
+ public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
Preconditions.checkNotNull(bucketInfo);
String volumeName = bucketInfo.getVolumeName();
String bucketName = bucketInfo.getBucketName();
@@ -155,9 +163,13 @@ public class BucketManagerImpl implements BucketManager {
if (bekb != null) {
omBucketInfoBuilder.setBucketEncryptionKey(bekb.build());
}
- metadataManager.getBucketTable().put(bucketKey,
- omBucketInfoBuilder.build());
+
+ OmBucketInfo omBucketInfo = omBucketInfoBuilder.build();
+ if (!isRatisEnabled) {
+ commitCreateBucketInfoToDB(omBucketInfo);
+ }
LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
+ return omBucketInfo;
} catch (IOException | DBException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Bucket creation failed for bucket:{} in volume:{}",
@@ -170,6 +182,27 @@ public class BucketManagerImpl implements BucketManager {
}
}
+
+ public void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException {
+ Preconditions.checkNotNull(omBucketInfo);
+ try {
+ commitCreateBucketInfoToDB(omBucketInfo);
+ } catch (IOException ex) {
+ LOG.error("Apply CreateBucket Failed for bucket: {}, volume: {}",
+ omBucketInfo.getBucketName(), omBucketInfo.getVolumeName(), ex);
+ throw ex;
+ }
+ }
+
+ private void commitCreateBucketInfoToDB(OmBucketInfo omBucketInfo)
+ throws IOException {
+ String dbBucketKey =
+ metadataManager.getBucketKey(omBucketInfo.getVolumeName(),
+ omBucketInfo.getBucketName());
+ metadataManager.getBucketTable().put(dbBucketKey,
+ omBucketInfo);
+ }
+
/**
* Returns Bucket Information.
*
@@ -210,7 +243,7 @@ public class BucketManagerImpl implements BucketManager {
* @throws IOException - On Failure.
*/
@Override
- public void setBucketProperty(OmBucketArgs args) throws IOException {
+ public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
Preconditions.checkNotNull(args);
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
@@ -262,8 +295,12 @@ public class BucketManagerImpl implements BucketManager {
}
bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
- metadataManager.getBucketTable()
- .put(bucketKey, bucketInfoBuilder.build());
+ OmBucketInfo omBucketInfo = bucketInfoBuilder.build();
+
+ if (!isRatisEnabled) {
+ commitSetBucketPropertyInfoToDB(omBucketInfo);
+ }
+ return omBucketInfo;
} catch (IOException | DBException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
@@ -275,6 +312,23 @@ public class BucketManagerImpl implements BucketManager {
}
}
+ public void applySetBucketProperty(OmBucketInfo omBucketInfo)
+ throws IOException {
+ try {
+ commitSetBucketPropertyInfoToDB(omBucketInfo);
+ } catch (IOException ex) {
+ LOG.error("Apply SetBucket property failed for bucket:{} in " +
+ "volume:{}", omBucketInfo.getBucketName(),
+ omBucketInfo.getVolumeName(), ex);
+ throw ex;
+ }
+ }
+
+ private void commitSetBucketPropertyInfoToDB(OmBucketInfo omBucketInfo)
+ throws IOException {
+ commitCreateBucketInfoToDB(omBucketInfo);
+ }
+
/**
* Updates the existing ACL list with remove and add ACLs that are passed.
* Remove is done before Add.
@@ -323,7 +377,10 @@ public class BucketManagerImpl implements BucketManager {
throw new OMException("Bucket is not empty",
OMException.ResultCodes.BUCKET_NOT_EMPTY);
}
- metadataManager.getBucketTable().delete(bucketKey);
+
+ if (!isRatisEnabled) {
+ commitDeleteBucketInfoToOMDB(bucketKey);
+ }
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName,
@@ -335,6 +392,25 @@ public class BucketManagerImpl implements BucketManager {
}
}
+ public void applyDeleteBucket(String volumeName, String bucketName)
+ throws IOException {
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ try {
+ commitDeleteBucketInfoToOMDB(metadataManager.getBucketKey(volumeName,
+ bucketName));
+ } catch (IOException ex) {
+ LOG.error("Apply DeleteBucket Failed for bucket: {}, volume: {}",
+ bucketName, volumeName, ex);
+ throw ex;
+ }
+ }
+
+ private void commitDeleteBucketInfoToOMDB(String dbBucketKey)
+ throws IOException {
+ metadataManager.getBucketTable().delete(dbBucketKey);
+ }
+
/**
* {@inheritDoc}
*/
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index e7f1e87..0d7dc80 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -279,6 +279,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private static String keyProviderUriKeyName =
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+
private OzoneManager(OzoneConfiguration conf) throws IOException,
AuthenticationException {
super(OzoneVersionInfo.OZONE_VERSION_INFO);
@@ -335,7 +336,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
LOG.error("Fail to create Key Provider");
}
- bucketManager = new BucketManagerImpl(metadataManager, getKmsProvider());
+ bucketManager = new BucketManagerImpl(metadataManager, getKmsProvider(),
+ isRatisEnabled);
metrics = OMMetrics.create();
s3BucketManager = new S3BucketManagerImpl(configuration, metadataManager,
@@ -1645,6 +1647,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return volumeList;
}
+ @Override
public void applyCreateVolume(OmVolumeArgs omVolumeArgs,
VolumeList volumeList) throws IOException {
// TODO: Need to add metrics and Audit log for HA requests
@@ -1707,6 +1710,66 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
+ @Override
+ public OmBucketInfo startCreateBucket(OmBucketInfo omBucketInfo)
+ throws IOException {
+ Preconditions.checkNotNull(omBucketInfo);
+ if(isAclEnabled) {
+ checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.CREATE,
+ omBucketInfo.getVolumeName(), omBucketInfo.getBucketName(), null);
+ }
+
+ return bucketManager.createBucket(omBucketInfo);
+ }
+
+ @Override
+ public void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException {
+ // TODO: Need to add metrics and Audit log for HA requests
+ bucketManager.applyCreateBucket(omBucketInfo);
+ }
+
+
+ @Override
+ public void startDeleteBucket(String volumeName, String bucketName)
+ throws IOException {
+ // TODO: Need to add metrics and Audit log for HA requests
+ if(isAclEnabled) {
+ checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.CREATE,
+ volumeName, bucketName, null);
+ }
+
+ bucketManager.deleteBucket(volumeName, bucketName);
+ }
+
+
+ @Override
+ public void applyDeleteBucket(String volumeName, String bucketName)
+ throws IOException {
+ // TODO: Need to add metrics and Audit log for HA requests
+ bucketManager.applyDeleteBucket(volumeName, bucketName);
+ }
+
+
+ @Override
+ public OmBucketInfo startSetBucketProperty(OmBucketArgs omBucketArgs)
+ throws IOException {
+ Preconditions.checkNotNull(omBucketArgs);
+ // TODO: Need to add metrics and Audit log for HA requests
+ if(isAclEnabled) {
+ checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.CREATE,
+ omBucketArgs.getVolumeName(), omBucketArgs.getBucketName(), null);
+ }
+ return bucketManager.setBucketProperty(omBucketArgs);
+ }
+
+
+ @Override
+ public void applySetBucketProperty(OmBucketInfo omBucketInfo)
+ throws IOException {
+ // TODO: Need to add metrics and Audit log for HA requests
+ bucketManager.applySetBucketProperty(omBucketInfo);
+ }
+
/**
* Checks if current caller has acl permissions.
*
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
index 440a45e..969f2c5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
@@ -141,7 +141,14 @@ public class S3BucketManagerImpl implements S3BucketManager {
throw new OMException("No such S3 bucket. " + bucketName,
OMException.ResultCodes.S3_BUCKET_NOT_FOUND);
}
- bucketManager.deleteBucket(getOzoneVolumeName(bucketName), bucketName);
+
+ if (isRatisEnabled) {
+ bucketManager.deleteBucket(getOzoneVolumeName(bucketName), bucketName);
+ bucketManager.applyDeleteBucket(getOzoneVolumeName(bucketName),
+ bucketName);
+ } else {
+ bucketManager.deleteBucket(getOzoneVolumeName(bucketName), bucketName);
+ }
omMetadataManager.getS3Table().delete(bucket);
} catch(IOException ex) {
throw ex;
@@ -200,7 +207,11 @@ public class S3BucketManagerImpl implements S3BucketManager {
.setIsVersionEnabled(Boolean.FALSE)
.setStorageType(StorageType.DEFAULT)
.build();
- bucketManager.createBucket(bucketInfo);
+ if (isRatisEnabled) {
+ bucketManager.applyCreateBucket(bucketManager.createBucket(bucketInfo));
+ } else {
+ bucketManager.createBucket(bucketInfo);
+ }
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
index 919709c..1c0b9a8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
@@ -192,6 +192,9 @@ public class OzoneManagerStateMachine extends BaseStateMachine {
case CreateVolume:
case SetVolumeProperty:
case DeleteVolume:
+ case CreateBucket:
+ case SetBucketProperty:
+ case DeleteBucket:
newOmRequest = handler.handleStartTransaction(omRequest);
break;
case AllocateBlock:
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java
index aada6e1..0830251 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java
@@ -19,16 +19,26 @@ package org.apache.hadoop.ozone.protocolPB;
import java.io.IOException;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDeleteVolumeResponse;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OmVolumeOwnerChangeResponse;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerServerProtocol;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateBucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateBucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.CreateVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.CreateVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .DeleteBucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .DeleteBucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.DeleteVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.DeleteVolumeResponse;
@@ -37,6 +47,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .SetBucketPropertyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .SetBucketPropertyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.SetVolumePropertyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.SetVolumePropertyResponse;
@@ -76,6 +90,15 @@ public class OzoneManagerHARequestHandlerImpl
case DeleteVolume:
newOmRequest = handleDeleteVolumeStart(omRequest);
break;
+ case CreateBucket:
+ newOmRequest = handleCreateBucketStart(omRequest);
+ break;
+ case SetBucketProperty:
+ newOmRequest = handleSetBucketPropertyStart(omRequest);
+ break;
+ case DeleteBucket:
+ newOmRequest = handleDeleteBucketRequestStart(omRequest);
+ break;
default:
throw new IOException("Unrecognized Command Type:" + cmdType);
}
@@ -104,6 +127,18 @@ public class OzoneManagerHARequestHandlerImpl
responseBuilder.setDeleteVolumeResponse(
handleDeleteVolumeApply(omRequest));
break;
+ case CreateBucket:
+ responseBuilder.setCreateBucketResponse(
+ handleCreateBucketApply(omRequest));
+ break;
+ case SetBucketProperty:
+ responseBuilder.setSetBucketPropertyResponse(
+ handleSetBucketPropertyApply(omRequest));
+ break;
+ case DeleteBucket:
+ responseBuilder.setDeleteBucketResponse(
+ handleDeleteBucketApply(omRequest));
+ break;
default:
// As all request types are not changed so we need to call handle
// here.
@@ -244,4 +279,91 @@ public class OzoneManagerHARequestHandlerImpl
return DeleteVolumeResponse.newBuilder().build();
}
+ private OMRequest handleCreateBucketStart(OMRequest omRequest)
+ throws IOException {
+
+ CreateBucketRequest createBucketRequest =
+ omRequest.getCreateBucketRequest();
+
+ OmBucketInfo omBucketInfo =
+ getOzoneManagerServerProtocol().startCreateBucket(
+ OmBucketInfo.getFromProtobuf(createBucketRequest.getBucketInfo()));
+
+ CreateBucketRequest newCreateBucketRequest =
+ CreateBucketRequest.newBuilder().setBucketInfo(
+ omBucketInfo.getProtobuf()).build();
+ return omRequest.toBuilder().setCreateBucketRequest(newCreateBucketRequest)
+ .build();
+
+ }
+
+
+ private CreateBucketResponse handleCreateBucketApply(OMRequest omRequest)
+ throws IOException {
+ CreateBucketRequest createBucketRequest =
+ omRequest.getCreateBucketRequest();
+
+ getOzoneManagerServerProtocol().applyCreateBucket(
+ OmBucketInfo.getFromProtobuf(createBucketRequest.getBucketInfo()));
+
+ return CreateBucketResponse.newBuilder().build();
+ }
+
+
+ private OMRequest handleDeleteBucketRequestStart(OMRequest omRequest)
+ throws IOException {
+
+ DeleteBucketRequest deleteBucketRequest =
+ omRequest.getDeleteBucketRequest();
+ getOzoneManagerServerProtocol().startDeleteBucket(
+ deleteBucketRequest.getVolumeName(),
+ deleteBucketRequest.getBucketName());
+
+ return omRequest;
+ }
+
+ private DeleteBucketResponse handleDeleteBucketApply(OMRequest omRequest)
+ throws IOException {
+
+ DeleteBucketRequest deleteBucketRequest =
+ omRequest.getDeleteBucketRequest();
+
+ getOzoneManagerServerProtocol().applyDeleteBucket(
+ deleteBucketRequest.getVolumeName(),
+ deleteBucketRequest.getBucketName());
+
+ return DeleteBucketResponse.newBuilder().build();
+ }
+
+ private OMRequest handleSetBucketPropertyStart(
+ OMRequest omRequest) throws IOException {
+ SetBucketPropertyRequest setBucketPropertyRequest =
+ omRequest.getSetBucketPropertyRequest();
+
+ OmBucketInfo omBucketInfo =
+ getOzoneManagerServerProtocol().startSetBucketProperty(
+ OmBucketArgs.getFromProtobuf(setBucketPropertyRequest.getBucketArgs()));
+
+ SetBucketPropertyRequest newSetBucketPropertyRequest =
+ SetBucketPropertyRequest.newBuilder()
+ .setBucketArgs(setBucketPropertyRequest.getBucketArgs())
+ .setBucketInfo(omBucketInfo.getProtobuf()).build();
+
+ return omRequest.toBuilder().setSetBucketPropertyRequest(
+ newSetBucketPropertyRequest).build();
+ }
+
+ private SetBucketPropertyResponse handleSetBucketPropertyApply(
+ OMRequest omRequest) throws IOException {
+ SetBucketPropertyRequest setBucketPropertyRequest =
+ omRequest.getSetBucketPropertyRequest();
+
+ getOzoneManagerServerProtocol().applySetBucketProperty(
+ OmBucketInfo.getFromProtobuf(setBucketPropertyRequest.getBucketInfo()));
+
+ return SetBucketPropertyResponse.newBuilder().build();
+ }
+
+
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: fix review comments
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch HDDS-1393
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8f65739bdb3ea24ec6217e3d3b51e07155e4b1de
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Mon Apr 8 16:53:44 2019 -0700
fix review comments
---
hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto | 5 ++++-
.../hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java | 1 -
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index e641464..2192767 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -478,7 +478,10 @@ message InfoBucketResponse {
}
message SetBucketPropertyRequest {
- required BucketArgs bucketArgs = 1;
+ //TODO: See if we can merge BucketArgs and BucketInfo.
+ optional BucketArgs bucketArgs = 1;
+ // Set during startTransaction and used in applyTransaction to apply to
+ // OM DB.
optional BucketInfo bucketInfo = 2;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java
index 0830251..9dd27b8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java
@@ -346,7 +346,6 @@ public class OzoneManagerHARequestHandlerImpl
SetBucketPropertyRequest newSetBucketPropertyRequest =
SetBucketPropertyRequest.newBuilder()
- .setBucketArgs(setBucketPropertyRequest.getBucketArgs())
.setBucketInfo(omBucketInfo.getProtobuf()).build();
return omRequest.toBuilder().setSetBucketPropertyRequest(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org