You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/05/27 16:15:15 UTC
[ozone] branch HDDS-3698-nonrolling-upgrade updated: HDDS-5244.
Allow multiple OM request versions to be supported at same layout version.
(#2265)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3698-nonrolling-upgrade by this push:
new 92b5106 HDDS-5244. Allow multiple OM request versions to be supported at same layout version. (#2265)
92b5106 is described below
commit 92b510653711e467c0c92fb54df6337ce77e7dec
Author: avijayanhwx <14...@users.noreply.github.com>
AuthorDate: Thu May 27 09:14:58 2021 -0700
HDDS-5244. Allow multiple OM request versions to be supported at same layout version. (#2265)
---
.github/workflows/post-commit.yml | 4 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 24 +--
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 2 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 189 +++++++++++++--------
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 42 ++++-
.../ozone/om/upgrade/OMLayoutVersionManager.java | 1 -
...OzoneManagerProtocolServerSideTranslatorPB.java | 6 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +-
.../hadoop/ozone/om/upgrade/MockOmRequest.java | 32 ++++
.../ozone/om/upgrade/OMLayoutFeatureUtil.java | 19 ++-
.../om/upgrade/TestOMLayoutFeatureAspect.java | 60 +++++--
.../ozone/om/upgrade/TestOMVersionManager.java | 25 +++
.../TestOmVersionManagerRequestFactory.java | 3 +
13 files changed, 290 insertions(+), 119 deletions(-)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index f2c8577..24f86e1 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -118,7 +118,7 @@ jobs:
acceptance:
needs: compile
runs-on: ubuntu-18.04
- timeout-minutes: 120
+ timeout-minutes: 120 # Todo HDDS-5270
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
strategy:
matrix:
@@ -163,7 +163,7 @@ jobs:
continue-on-error: true
integration:
runs-on: ubuntu-18.04
- timeout-minutes: 100
+ timeout-minutes: 120
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
strategy:
matrix:
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
index 68d0715..05ba8c4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus;
import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
/**
@@ -74,28 +75,24 @@ public class TestOzoneManagerPrepare extends TestOzoneManagerHA {
}
/**
- * Calls prepare on all OMs when they have no transaction information.
- * Checks that they are brought into prepare mode successfully.
+ * Writes data to the cluster via the leader OM, and then prepares it.
+ * Checks that every OM is prepared successfully.
*/
@Test
- public void testPrepareWithoutTransactions() throws Exception {
+ public void testPrepareWithTransactions() throws Exception {
setup();
+
long prepareIndex = submitPrepareRequest();
assertClusterPrepared(prepareIndex);
assertRatisLogsCleared();
- }
- /**
- * Writes data to the cluster via the leader OM, and then prepares it.
- * Checks that every OM is prepared successfully.
- */
- @Test
- public void testPrepareWithTransactions() throws Exception {
- setup();
+ submitCancelPrepareRequest();
+ assertClusterNotPrepared();
+
String volumeName = VOLUME + UUID.randomUUID().toString();
Set<String> writtenKeys = writeKeysAndWaitForLogs(volumeName, 50,
cluster.getOzoneManagersList());
- long prepareIndex = submitPrepareRequest();
+ prepareIndex = submitPrepareRequest();
// Make sure all OMs are prepared and all OMs still have their data.
assertClusterPrepared(prepareIndex);
@@ -171,6 +168,7 @@ public class TestOzoneManagerPrepare extends TestOzoneManagerHA {
}
}
+ @Ignore("Flaky test tracked in HDDS-5109")
@Test
public void testPrepareWithRestart() throws Exception {
setup();
@@ -188,6 +186,8 @@ public class TestOzoneManagerPrepare extends TestOzoneManagerHA {
assertClusterPrepared(prepareIndex);
}
+ @Ignore("Saving on CI time since this is a pessimistic test. We should not " +
+ "be able to do anything with 2 OMs down.")
@Test
public void testPrepareFailsWhenTwoOmsAreDown() throws Exception {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
index 7d2abdf..6d7a88a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
@@ -109,7 +109,7 @@ public class TrashOzoneFileSystem extends FileSystem {
ozoneManager.getMetrics().incNumTrashWriteRequests();
if (ozoneManager.isRatisEnabled()) {
OMClientRequest omClientRequest =
- OzoneManagerRatisUtils.getRequest(ozoneManager, omRequest);
+ OzoneManagerRatisUtils.createClientRequest(omRequest);
omRequest = omClientRequest.preExecute(ozoneManager);
RaftClientRequest req = getRatisRequest(omRequest);
ozoneManager.getOmRatisServer().submitRequest(omRequest, req);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 20ee3a5..6925d0e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -18,23 +18,49 @@
package org.apache.hadoop.ozone.om.ratis.utils;
import com.google.common.base.Preconditions;
-
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.hdds.utils.TransactionInfo;
+import org.apache.hadoop.ozone.om.request.bucket.OMBucketCreateRequest;
+import org.apache.hadoop.ozone.om.request.bucket.OMBucketDeleteRequest;
+import org.apache.hadoop.ozone.om.request.bucket.OMBucketSetPropertyRequest;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.bucket.acl.OMBucketAddAclRequest;
import org.apache.hadoop.ozone.om.request.bucket.acl.OMBucketRemoveAclRequest;
import org.apache.hadoop.ozone.om.request.bucket.acl.OMBucketSetAclRequest;
+import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequest;
+import org.apache.hadoop.ozone.om.request.file.OMFileCreateRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeysDeleteRequest;
+import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest;
+import org.apache.hadoop.ozone.om.request.key.OMKeysRenameRequest;
+import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyRemoveAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeySetAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixAddAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixRemoveAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest;
+import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequest;
+import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequest;
+import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequest;
+import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCompleteRequest;
+import org.apache.hadoop.ozone.om.request.s3.security.S3GetSecretRequest;
+import org.apache.hadoop.ozone.om.request.security.OMCancelDelegationTokenRequest;
+import org.apache.hadoop.ozone.om.request.security.OMGetDelegationTokenRequest;
+import org.apache.hadoop.ozone.om.request.security.OMRenewDelegationTokenRequest;
+import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest;
+import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest;
+import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest;
+import org.apache.hadoop.ozone.om.request.volume.OMVolumeCreateRequest;
+import org.apache.hadoop.ozone.om.request.volume.OMVolumeDeleteRequest;
import org.apache.hadoop.ozone.om.request.volume.OMVolumeSetOwnerRequest;
import org.apache.hadoop.ozone.om.request.volume.OMVolumeSetQuotaRequest;
import org.apache.hadoop.ozone.om.request.volume.acl.OMVolumeAddAclRequest;
@@ -46,8 +72,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneOb
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.rocksdb.RocksDBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
@@ -58,110 +82,133 @@ import java.nio.file.Path;
*/
public final class OzoneManagerRatisUtils {
- private static final Logger LOG = LoggerFactory
- .getLogger(OzoneManagerRatisUtils.class);
-
private OzoneManagerRatisUtils() {
}
-
- public static OMClientRequest getRequest(OzoneManager om,
- OMRequest omRequest) {
+ /**
+ * Create OMClientRequest which encapsulates the OMRequest.
+ * @param omRequest
+ * @return OMClientRequest
+ * @throws IOException
+ */
+ public static OMClientRequest createClientRequest(OMRequest omRequest) {
Type cmdType = omRequest.getCmdType();
switch (cmdType) {
+ case CreateVolume:
+ return new OMVolumeCreateRequest(omRequest);
+ case SetVolumeProperty:
+ boolean hasQuota = omRequest.getSetVolumePropertyRequest()
+ .hasQuotaInBytes();
+ boolean hasOwner = omRequest.getSetVolumePropertyRequest().hasOwnerName();
+ Preconditions.checkState(hasOwner || hasQuota, "Either Quota or owner " +
+ "should be set in the SetVolumeProperty request");
+ Preconditions.checkState(!(hasOwner && hasQuota), "Either Quota or " +
+ "owner should be set in the SetVolumeProperty request. Should not " +
+ "set both");
+ if (hasQuota) {
+ return new OMVolumeSetQuotaRequest(omRequest);
+ } else {
+ return new OMVolumeSetOwnerRequest(omRequest);
+ }
+ case DeleteVolume:
+ return new OMVolumeDeleteRequest(omRequest);
+ case CreateBucket:
+ return new OMBucketCreateRequest(omRequest);
+ case DeleteBucket:
+ return new OMBucketDeleteRequest(omRequest);
+ case SetBucketProperty:
+ return new OMBucketSetPropertyRequest(omRequest);
+ case AllocateBlock:
+ return new OMAllocateBlockRequest(omRequest);
+ case CreateKey:
+ return new OMKeyCreateRequest(omRequest);
+ case CommitKey:
+ return new OMKeyCommitRequest(omRequest);
+ case DeleteKey:
+ return new OMKeyDeleteRequest(omRequest);
+ case DeleteKeys:
+ return new OMKeysDeleteRequest(omRequest);
+ case RenameKey:
+ return new OMKeyRenameRequest(omRequest);
+ case RenameKeys:
+ return new OMKeysRenameRequest(omRequest);
+ case CreateDirectory:
+ return new OMDirectoryCreateRequest(omRequest);
+ case CreateFile:
+ return new OMFileCreateRequest(omRequest);
+ case PurgeKeys:
+ return new OMKeyPurgeRequest(omRequest);
+ case InitiateMultiPartUpload:
+ return new S3InitiateMultipartUploadRequest(omRequest);
+ case CommitMultiPartUpload:
+ return new S3MultipartUploadCommitPartRequest(omRequest);
+ case AbortMultiPartUpload:
+ return new S3MultipartUploadAbortRequest(omRequest);
+ case CompleteMultiPartUpload:
+ return new S3MultipartUploadCompleteRequest(omRequest);
case AddAcl:
case RemoveAcl:
case SetAcl:
- return getOMAclRequest(om, omRequest);
- case SetVolumeProperty:
- return getVolumeSetPropertyRequest(om, omRequest);
+ return getOMAclRequest(omRequest);
+ case GetDelegationToken:
+ return new OMGetDelegationTokenRequest(omRequest);
+ case CancelDelegationToken:
+ return new OMCancelDelegationTokenRequest(omRequest);
+ case RenewDelegationToken:
+ return new OMRenewDelegationTokenRequest(omRequest);
+ case GetS3Secret:
+ return new S3GetSecretRequest(omRequest);
+ case RecoverTrash:
+ return new OMTrashRecoverRequest(omRequest);
+ case FinalizeUpgrade:
+ return new OMFinalizeUpgradeRequest(omRequest);
+ case Prepare:
+ return new OMPrepareRequest(omRequest);
+ case CancelPrepare:
+ return new OMCancelPrepareRequest(omRequest);
default:
- Class<? extends OMClientRequest> requestClass =
- om.getVersionManager()
- .getHandler(omRequest.getCmdType().name());
- return getClientRequest(requestClass, omRequest);
- }
- }
-
- private static OMClientRequest getClientRequest(Class<?
- extends OMClientRequest> requestClass, OMRequest omRequest) {
- try {
- return requestClass.getDeclaredConstructor(OMRequest.class)
- .newInstance(omRequest);
- } catch (Exception ex) {
- LOG.error("Unable to get request handler for '{}', current layout " +
- "version = {}, request factory returned '{}'",
- omRequest.getCmdType(),
- omRequest.getLayoutVersion().getVersion(),
- requestClass.getSimpleName());
+ throw new IllegalStateException("Unrecognized write command " +
+ "type request" + cmdType);
}
- throw new IllegalStateException("Unrecognized write command " +
- "type request : " + omRequest.getCmdType());
}
- public static OMClientRequest getOMAclRequest(OzoneManager om,
- OMRequest omRequest) {
+ private static OMClientRequest getOMAclRequest(OMRequest omRequest) {
Type cmdType = omRequest.getCmdType();
- String requestType = null;
if (Type.AddAcl == cmdType) {
ObjectType type = omRequest.getAddAclRequest().getObj().getResType();
if (ObjectType.VOLUME == type) {
- requestType = OMVolumeAddAclRequest.getRequestType();
+ return new OMVolumeAddAclRequest(omRequest);
} else if (ObjectType.BUCKET == type) {
- requestType = OMBucketAddAclRequest.getRequestType();
+ return new OMBucketAddAclRequest(omRequest);
} else if (ObjectType.KEY == type) {
- requestType = OMKeyAddAclRequest.getRequestType();
+ return new OMKeyAddAclRequest(omRequest);
} else {
- requestType = OMPrefixAddAclRequest.getRequestType();
+ return new OMPrefixAddAclRequest(omRequest);
}
} else if (Type.RemoveAcl == cmdType) {
ObjectType type = omRequest.getRemoveAclRequest().getObj().getResType();
if (ObjectType.VOLUME == type) {
- requestType = OMVolumeRemoveAclRequest.getRequestType();
+ return new OMVolumeRemoveAclRequest(omRequest);
} else if (ObjectType.BUCKET == type) {
- requestType = OMBucketRemoveAclRequest.getRequestType();
+ return new OMBucketRemoveAclRequest(omRequest);
} else if (ObjectType.KEY == type) {
- requestType = OMKeyRemoveAclRequest.getRequestType();
+ return new OMKeyRemoveAclRequest(omRequest);
} else {
- requestType = OMPrefixRemoveAclRequest.getRequestType();
+ return new OMPrefixRemoveAclRequest(omRequest);
}
} else {
ObjectType type = omRequest.getSetAclRequest().getObj().getResType();
if (ObjectType.VOLUME == type) {
- requestType = OMVolumeSetAclRequest.getRequestType();
+ return new OMVolumeSetAclRequest(omRequest);
} else if (ObjectType.BUCKET == type) {
- requestType = OMBucketSetAclRequest.getRequestType();
+ return new OMBucketSetAclRequest(omRequest);
} else if (ObjectType.KEY == type) {
- requestType = OMKeySetAclRequest.getRequestType();
+ return new OMKeySetAclRequest(omRequest);
} else {
- requestType = OMPrefixSetAclRequest.getRequestType();
+ return new OMPrefixSetAclRequest(omRequest);
}
}
- Class<? extends OMClientRequest> requestClass =
- om.getVersionManager().getHandler(requestType);
- return getClientRequest(requestClass, omRequest);
}
- public static OMClientRequest getVolumeSetPropertyRequest(
- OzoneManager om, OMRequest omRequest) {
- boolean hasQuota = omRequest.getSetVolumePropertyRequest()
- .hasQuotaInBytes();
- boolean hasOwner = omRequest.getSetVolumePropertyRequest().hasOwnerName();
- Preconditions.checkState(hasOwner || hasQuota,
- "Either Quota or owner " +
- "should be set in the SetVolumeProperty request");
- Preconditions.checkState(!(hasOwner && hasQuota),
- "Either Quota or " +
- "owner should be set in the SetVolumeProperty request. Should not "
- + "set both");
-
- String requestType = hasQuota ? OMVolumeSetQuotaRequest.getRequestType() :
- OMVolumeSetOwnerRequest.getRequestType();
- Class<? extends OMClientRequest> requestClass =
- om.getVersionManager().getHandler(requestType);
- return getClientRequest(requestClass, omRequest);
- }
-
-
/**
* Convert exception result to {@link OzoneManagerProtocolProtos.Status}.
* @param exception
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java
index ad0af55..a6fe773 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.om.upgrade;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
+import java.io.IOException;
import java.lang.reflect.Method;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -30,8 +31,10 @@ import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
+import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* 'Aspect' for OM Layout Feature API. All methods annotated with the
@@ -43,9 +46,11 @@ public class OMLayoutFeatureAspect {
public static final String GET_VERSION_MANAGER_METHOD_NAME =
"getOmVersionManager";
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OMLayoutFeatureAspect.class);
@Before("@annotation(DisallowedUntilLayoutVersion) && execution(* *(..))")
- public void checkLayoutFeature(JoinPoint joinPoint) throws Throwable {
+ public void checkLayoutFeature(JoinPoint joinPoint) throws IOException {
String featureName = ((MethodSignature) joinPoint.getSignature())
.getMethod().getAnnotation(DisallowedUntilLayoutVersion.class)
.value().name();
@@ -63,16 +68,47 @@ public class OMLayoutFeatureAspect {
lvm = new OMLayoutVersionManager();
}
}
+ checkIsAllowed(joinPoint.getSignature().toShortString(), lvm, featureName);
+ }
+
+ private void checkIsAllowed(String operationName,
+ LayoutVersionManager lvm,
+ String featureName) throws OMException {
if (!lvm.isAllowed(featureName)) {
LayoutFeature layoutFeature = lvm.getFeature(featureName);
throw new OMException(String.format("Operation %s cannot be invoked " +
"before finalization. It belongs to the layout feature %s, " +
"whose layout version is %d. Current Layout version is %d",
- joinPoint.getSignature().toShortString(),
+ operationName,
layoutFeature.name(),
layoutFeature.layoutVersion(),
lvm.getMetadataLayoutVersion()),
NOT_SUPPORTED_OPERATION);
}
}
+
+ @Pointcut("execution(* " +
+ "org.apache.hadoop.ozone.om.request.OMClientRequest+.preExecute(..)) " +
+ "&& @this(org.apache.hadoop.ozone.om.upgrade.BelongsToLayoutVersion)")
+ public void omRequestPointCut() {
+ }
+
+ @Before("omRequestPointCut()")
+ public void beforeRequestApplyTxn(final JoinPoint joinPoint)
+ throws OMException {
+
+ BelongsToLayoutVersion annotation = joinPoint.getTarget().getClass()
+ .getAnnotation(BelongsToLayoutVersion.class);
+ if (annotation == null) {
+ return;
+ }
+
+ Object[] args = joinPoint.getArgs();
+ OzoneManager om = (OzoneManager) args[0];
+
+ LayoutFeature lf = annotation.value();
+ checkIsAllowed(joinPoint.getTarget().getClass().getSimpleName(),
+ om.getVersionManager(), lf.name());
+ }
+
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
index fd9f7a1..449e340 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
@@ -88,7 +88,6 @@ public final class OMLayoutVersionManager
NOT_SUPPORTED_OPERATION);
}
registerUpgradeActions(OM_UPGRADE_CLASS_PACKAGE);
- registerOzoneManagerRequests(OM_REQUEST_CLASS_PACKAGE);
}
/**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index bdf11da..8ecf53c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.protocolPB;
-import static org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils.getRequest;
+import static org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils.createClientRequest;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.PrepareStatus;
import java.io.IOException;
@@ -136,7 +136,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
raftServerStatus = omRatisServer.checkLeaderStatus();
if (raftServerStatus == LEADER_AND_READY) {
try {
- OMClientRequest omClientRequest = getRequest(ozoneManager, request);
+ OMClientRequest omClientRequest = createClientRequest(request);
request = omClientRequest.preExecute(ozoneManager);
} catch (IOException ex) {
// As some of the preExecute returns error. So handle here.
@@ -241,7 +241,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
if (OmUtils.isReadOnly(request)) {
return handler.handleReadRequest(request);
} else {
- OMClientRequest omClientRequest = getRequest(ozoneManager, request);
+ OMClientRequest omClientRequest = createClientRequest(request);
request = omClientRequest.preExecute(ozoneManager);
index = transactionIndex.incrementAndGet();
omClientResponse = handler.handleWriteRequest(request, index);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index bbb0696..e1ccce8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -240,7 +240,7 @@ public class OzoneManagerRequestHandler implements RequestHandler {
public OMClientResponse handleWriteRequest(OMRequest omRequest,
long transactionLogIndex) {
OMClientRequest omClientRequest =
- OzoneManagerRatisUtils.getRequest(getOzoneManager(), omRequest);
+ OzoneManagerRatisUtils.createClientRequest(omRequest);
OMClientResponse omClientResponse =
omClientRequest.validateAndUpdateCache(getOzoneManager(),
transactionLogIndex, ozoneManagerDoubleBuffer::add);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java
new file mode 100644
index 0000000..f811dfc
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION;
+
+import org.apache.hadoop.ozone.om.OzoneManager;
+
+/**
+ * Mock OM Request class for testing annotation.
+ */
+@BelongsToLayoutVersion(INITIAL_VERSION)
+public class MockOmRequest {
+ public void preExecute(OzoneManager om) {
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureUtil.java
index edcc356..2359127 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureUtil.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureUtil.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.ozone.om.upgrade;
+import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
/**
@@ -36,7 +38,7 @@ public class OMLayoutFeatureUtil {
* "disallowed" by just adding the following annotation, thereby keeping the
* method logic and upgrade logic separate.
*/
- //@DisallowedUntilLayoutVersion(ERASURE_CODING)
+ @DisallowedUntilLayoutVersion(INITIAL_VERSION)
public String ecMethod() {
// Blah Blah EC Blah....
return "ec";
@@ -54,11 +56,10 @@ public class OMLayoutFeatureUtil {
// A method named 'getOmVersionManager' needed for the Aspect to get
// instance of the layout version manager.
public LayoutVersionManager getOmVersionManager() throws IOException {
- OzoneConfiguration configuration = new OzoneConfiguration();
- Path tempDirWithPrefix = Files.createTempDirectory("TestAspect");
- configuration.set("ozone.metadata.dirs",
- tempDirWithPrefix.toAbsolutePath().toString());
- return new OMLayoutVersionManager(0);
+ LayoutVersionManager mockLvm = mock(LayoutVersionManager.class);
+ when(mockLvm.isAllowed(anyString())).thenReturn(false).thenReturn(true);
+ when(mockLvm.getFeature(anyString())).thenReturn(INITIAL_VERSION);
+ return mockLvm;
}
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java
index bbfb059..33cfc99 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java
@@ -18,16 +18,20 @@
package org.apache.hadoop.ozone.om.upgrade;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
-import static org.junit.Assert.assertEquals;
+import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.junit.Assert;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.reflect.MethodSignature;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -54,18 +58,42 @@ public class TestOMLayoutFeatureAspect {
* should fail, and the second one should pass.
* @throws Exception
*/
- @Ignore
@Test
- public void testCheckLayoutFeature() throws Exception {
+ public void testDisallowedUntilLayoutVersion() throws Throwable {
OMLayoutFeatureUtil testObj = new OMLayoutFeatureUtil();
- try {
- String s = testObj.ecMethod();
- Assert.fail();
- } catch (Exception ex) {
- OMException omEx = (OMException) ex;
- assertEquals(NOT_SUPPORTED_OPERATION, omEx.getResult());
- }
- String s = testObj.basicMethod();
- assertEquals("basic", s);
+ OMLayoutFeatureAspect aspect = new OMLayoutFeatureAspect();
+
+ JoinPoint joinPoint = mock(JoinPoint.class);
+ when(joinPoint.getTarget()).thenReturn(testObj);
+
+ MethodSignature methodSignature = mock(MethodSignature.class);
+ when(methodSignature.getMethod())
+ .thenReturn(OMLayoutFeatureUtil.class.getMethod("ecMethod"));
+ when(joinPoint.getSignature()).thenReturn(methodSignature);
+
+ LambdaTestUtils.intercept(OMException.class,
+ "cannot be invoked before finalization",
+ () -> aspect.checkLayoutFeature(joinPoint));
+ }
+
+ @Test
+ public void testPreExecuteLayoutCheck() throws Exception {
+
+ OzoneManager om = mock(OzoneManager.class);
+ OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
+ when(lvm.isAllowed(anyString())).thenReturn(false);
+ when(lvm.getFeature(anyString())).thenReturn(INITIAL_VERSION);
+ when(om.getVersionManager()).thenReturn(lvm);
+
+ MockOmRequest mockOmRequest = new MockOmRequest();
+ OMLayoutFeatureAspect aspect = new OMLayoutFeatureAspect();
+
+ JoinPoint joinPoint = mock(JoinPoint.class);
+ when(joinPoint.getArgs()).thenReturn(new Object[]{om});
+ when(joinPoint.getTarget()).thenReturn(mockOmRequest);
+
+ LambdaTestUtils.intercept(OMException.class,
+ "cannot be invoked before finalization",
+ () -> aspect.beforeRequestApplyTxn(joinPoint));
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
index 5c32979..02d581d 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
@@ -37,6 +37,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@@ -46,6 +47,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
/**
@@ -101,6 +103,9 @@ public class TestOMVersionManager {
verify(omMock, times(UpgradeActionType.values().length)).getVersion();
}
+ @Ignore("Since there is no longer a need to enforce the getRequestType " +
+ "method in OM request classes, disabling the " +
+ "test. Potentially revisit later.")
@Test
public void testAllOMRequestClassesHaveRequestType()
throws InvocationTargetException, IllegalAccessException {
@@ -133,6 +138,26 @@ public class TestOMVersionManager {
}
@Test
+ /*
+ * The OMLayoutFeatureAspect relies on the fact that the OM client
+ * request handler class has a preExecute method with first argument as
+ * 'OzoneManager'. If that is not true, please fix
+ * OMLayoutFeatureAspect#beforeRequestApplyTxn.
+ */
+ public void testOmClientRequestPreExecuteIsCompatibleWithAspect() {
+ Method[] methods = OMClientRequest.class.getMethods();
+
+ Optional<Method> preExecuteMethod = Arrays.stream(methods)
+ .filter(m -> m.getName().equals("preExecute"))
+ .findFirst();
+
+ assertTrue(preExecuteMethod.isPresent());
+ assertTrue(preExecuteMethod.get().getParameterCount() >= 1);
+ assertEquals(OzoneManager.class,
+ preExecuteMethod.get().getParameterTypes()[0]);
+ }
+
+ @Test
public void testOmUpgradeActionsRegistered() throws Exception {
OMLayoutVersionManager lvm = new OMLayoutVersionManager(); // MLV >= 0
assertFalse(lvm.needsFinalization());
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOmVersionManagerRequestFactory.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOmVersionManagerRequestFactory.java
index 4966c21..44afda5 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOmVersionManagerRequestFactory.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOmVersionManagerRequestFactory.java
@@ -33,12 +33,15 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.reflections.Reflections;
/**
* Test OmVersionFactory.
*/
+@Ignore("Ignored since this is incompatible with HDDS-2939 work. Potentially " +
+ "revisit later.")
public class TestOmVersionManagerRequestFactory {
private static OMLayoutVersionManager omVersionManager;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org