You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2020/07/02 13:21:57 UTC
[GitHub] [hadoop-ozone] adoroszlai commented on a change in pull request #1150: HDDS-3903. OzoneRpcClient support batch rename keys.
adoroszlai commented on a change in pull request #1150:
URL: https://github.com/apache/hadoop-ozone/pull/1150#discussion_r448995364
##########
File path: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
##########
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.OmRenameKeyInfo;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteResponse;
+import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysResponse;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+
+/**
+ * Handles rename keys request.
+ */
+public class OMKeysRenameRequest extends OMKeyRequest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OMKeysRenameRequest.class);
+
+ public OMKeysRenameRequest(OMRequest omRequest) {
+ super(omRequest);
+ }
+
+ /**
+ * Stores the result of request execution for Rename Requests.
+ */
+ private enum Result {
+ SUCCESS,
+ DELETE_FROM_KEY_ONLY,
+ REPLAY,
+ FAILURE,
+ }
+
+ @Override
+ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
+
+ RenameKeysRequest renameKeys = getOmRequest().getRenameKeysRequest();
+ Preconditions.checkNotNull(renameKeys);
+
+ List<RenameKeyRequest> renameKeyList = new ArrayList<>();
+ for (RenameKeyRequest renameKey : renameKeys.getRenameKeyRequestList()) {
+ // Set modification time.
+ KeyArgs.Builder newKeyArgs = renameKey.getKeyArgs().toBuilder()
+ .setModificationTime(Time.now());
+ renameKey.toBuilder().setKeyArgs(newKeyArgs);
+ renameKeyList.add(renameKey);
+ }
+ RenameKeysRequest renameKeysRequest = RenameKeysRequest
+ .newBuilder().addAllRenameKeyRequest(renameKeyList).build();
+ return getOmRequest().toBuilder().setRenameKeysRequest(renameKeysRequest)
+ .setUserInfo(getUserInfo()).build();
+ }
+
+ @Override
+ @SuppressWarnings("methodlength")
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+
+ RenameKeysRequest renameKeysRequest = getOmRequest().getRenameKeysRequest();
+ OMClientResponse omClientResponse = null;
+ Set<OmKeyInfo> unRenamedKeys = new HashSet<>();
+ List<OmRenameKeyInfo> renameKeyInfoList = new ArrayList<>();
+
+ OMMetrics omMetrics = ozoneManager.getMetrics();
+ omMetrics.incNumKeyRenames();
+
+ AuditLogger auditLogger = ozoneManager.getAuditLogger();
+
+
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
+
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+ IOException exception = null;
+ OmKeyInfo fromKeyValue = null;
+
+ Result result = null;
+ Map<String, String> auditMap = null;
+ RenameKeyRequest renameRequest = null;
+ String toKey = null;
+ String fromKey = null;
+ String volumeName = null;
+ String bucketName = null;
+ String fromKeyName = null;
+ String toKeyName = null;
+ try {
+ for (RenameKeyRequest renameKeyRequest : renameKeysRequest
+ .getRenameKeyRequestList()) {
+ OzoneManagerProtocolProtos.KeyArgs renameKeyArgs =
+ renameKeyRequest.getKeyArgs();
+ volumeName = renameKeyArgs.getVolumeName();
+ bucketName = renameKeyArgs.getBucketName();
+ fromKeyName = renameKeyArgs.getKeyName();
+ String objectKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
+ fromKeyName);
+ OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable().get(objectKey);
+ unRenamedKeys.add(omKeyInfo);
+ }
+
+ for (RenameKeyRequest renameKeyRequest : renameKeysRequest
+ .getRenameKeyRequestList()) {
+ OzoneManagerProtocolProtos.KeyArgs renameKeyArgs =
+ renameKeyRequest.getKeyArgs();
+
+ volumeName = renameKeyArgs.getVolumeName();
+ bucketName = renameKeyArgs.getBucketName();
+ fromKeyName = renameKeyArgs.getKeyName();
+ toKeyName = renameKeyRequest.getToKeyName();
+ auditMap = buildAuditMap(renameKeyArgs, renameKeyRequest);
+ renameRequest = renameKeyRequest;
+
+ if (toKeyName.length() == 0 || fromKeyName.length() == 0) {
+ throw new OMException("Key name is empty",
+ OMException.ResultCodes.INVALID_KEY_NAME);
+ }
+ // check Acls to see if user has access to perform delete operation on
+ // old key and create operation on new key
+ checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName,
+ IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
+ IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
+
+ // Validate bucket and volume exists or not.
+ validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
+
+ // Check if toKey exists
+ fromKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
+ fromKeyName);
+ toKey =
+ omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName);
+ OmKeyInfo toKeyValue = omMetadataManager.getKeyTable().get(toKey);
+
+ if (toKeyValue != null) {
+
+ // Check if this transaction is a replay of ratis logs.
+ if (isReplay(ozoneManager, toKeyValue, trxnLogIndex)) {
+
+ // Check if fromKey is still in the DB and created before this
+ // replay.
+ // For example, lets say we have the following sequence of
+ // transactions.
+ // Trxn 1 : Create Key1
+ // Trnx 2 : Rename Key1 to Key2 -> Deletes Key1 and Creates Key2
+ // Now if these transactions are replayed:
+ // Replay Trxn 1 : Creates Key1 again it does not exist in DB
+ // Replay Trxn 2 : Key2 is not created as it exists in DB and
+ // the request would be deemed a replay. But
+ // Key1 is still in the DB and needs to be
+ // deleted.
+ fromKeyValue = omMetadataManager.getKeyTable().get(fromKey);
+ if (fromKeyValue != null) {
+ // Check if this replay transaction was after the fromKey was
+ // created. If so, we have to delete the fromKey.
+ if (ozoneManager.isRatisEnabled() &&
+ trxnLogIndex > fromKeyValue.getUpdateID()) {
+ // Add to cache. Only fromKey should be deleted. ToKey already
+ // exists in DB as this transaction is a replay.
+ result = Result.DELETE_FROM_KEY_ONLY;
+ renameKeyInfoList.add(new OmRenameKeyInfo(
+ null, fromKeyValue));
+ }
+ }
+
+ if (result == null) {
+ result = Result.REPLAY;
+ // If toKey exists and fromKey does not, then no further action is
+ // required. Return a dummy OMClientResponse.
+ omClientResponse =
+ new OMKeysRenameResponse(createReplayOMResponse(
+ omResponse));
+ }
+ } else {
+ // This transaction is not a replay. toKeyName should not exist
+ throw new OMException("Key already exists " + toKeyName,
+ OMException.ResultCodes.KEY_ALREADY_EXISTS);
+ }
+ } else {
+ // fromKeyName should exist
+ fromKeyValue = omMetadataManager.getKeyTable().get(fromKey);
+ if (fromKeyValue == null) {
+ // TODO: Add support for renaming open key
+ throw new OMException("Key not found " + fromKey, KEY_NOT_FOUND);
+ }
+
+ fromKeyValue.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
+
+ fromKeyValue.setKeyName(toKeyName);
+ //Set modification time
+ fromKeyValue.setModificationTime(renameKeyArgs.getModificationTime());
+
+ renameKeyInfoList
+ .add(new OmRenameKeyInfo(fromKeyName, fromKeyValue));
+ }
+ }
+ omClientResponse = new OMKeysRenameResponse(omResponse
+ .setRenameKeysResponse(RenameKeysResponse.newBuilder()).build(),
+ renameKeyInfoList, trxnLogIndex);
+ result = Result.SUCCESS;
+ } catch (IOException ex) {
+ result = Result.FAILURE;
+ exception = ex;
+ omClientResponse = new OMKeyDeleteResponse(
+ createRenameKeysErrorOMResponse(omResponse, exception,
+ unRenamedKeys));
+ } finally {
+ addResponseToDoubleBuffer(trxnLogIndex, omClientResponse,
+ omDoubleBufferHelper);
+ }
+
+ if (result == Result.SUCCESS || result == Result.FAILURE) {
+ auditLog(auditLogger, buildAuditMessage(OMAction.RENAME_KEY, auditMap,
+ exception, getOmRequest().getUserInfo()));
Review comment:
Shouldn't all renamed keys be logged to audit?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org