You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ms...@apache.org on 2018/05/09 13:37:01 UTC
hadoop git commit: HDDS-20. Ozone: Add support for rename key within
a bucket for rpc client. Contributed by Lokesh Jain.
Repository: hadoop
Updated Branches:
refs/heads/trunk cd68c7cc6 -> 208b97e96
HDDS-20. Ozone: Add support for rename key within a bucket for rpc client. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/208b97e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/208b97e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/208b97e9
Branch: refs/heads/trunk
Commit: 208b97e969cda59827890dfd26f53e3b7f20d7de
Parents: cd68c7c
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Wed May 9 19:06:07 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Wed May 9 19:06:07 2018 +0530
----------------------------------------------------------------------
.../apache/hadoop/ozone/client/OzoneBucket.java | 8 ++
.../ozone/client/protocol/ClientProtocol.java | 10 ++
.../hadoop/ozone/client/rest/RestClient.java | 6 +
.../hadoop/ozone/client/rpc/RpcClient.java | 15 +++
.../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 6 +-
.../ksm/protocol/KeySpaceManagerProtocol.java | 9 +-
...ceManagerProtocolClientSideTranslatorPB.java | 27 +++++
.../main/proto/KeySpaceManagerProtocol.proto | 20 +++-
.../ozone/client/rpc/TestOzoneRpcClient.java | 44 +++++++
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 115 +++++++++++++++++++
.../ozone/web/interfaces/StorageHandler.java | 9 ++
.../web/localstorage/LocalStorageHandler.java | 6 +
.../web/storage/DistributedStorageHandler.java | 11 ++
.../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 22 ++++
.../org/apache/hadoop/ozone/ksm/KeyManager.java | 10 ++
.../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 65 +++++++++++
.../hadoop/ozone/ksm/KeySpaceManager.java | 11 ++
.../ozone/ksm/exceptions/KSMException.java | 2 +
...ceManagerProtocolServerSideTranslatorPB.java | 26 +++++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 29 ++---
20 files changed, 427 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index ba6286b..1712979 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -305,6 +305,14 @@ public class OzoneBucket {
proxy.deleteKey(volumeName, name, key);
}
+ public void renameKey(String fromKeyName, String toKeyName)
+ throws IOException {
+ Preconditions.checkNotNull(proxy, "Client proxy is not set.");
+ Preconditions.checkNotNull(fromKeyName);
+ Preconditions.checkNotNull(toKeyName);
+ proxy.renameKey(volumeName, name, fromKeyName, toKeyName);
+ }
+
/**
* An Iterator to iterate over {@link OzoneKey} list.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 816c185..94cc257 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -285,6 +285,16 @@ public interface ClientProtocol {
void deleteKey(String volumeName, String bucketName, String keyName)
throws IOException;
+ /**
+ * Renames an existing key within a bucket.
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param fromKeyName Name of the Key to be renamed
+ * @param toKeyName New name to be used for the Key
+ * @throws IOException
+ */
+ void renameKey(String volumeName, String bucketName, String fromKeyName,
+ String toKeyName) throws IOException;
/**
* Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index b8b4610..e9885d1 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -676,6 +676,12 @@ public class RestClient implements ClientProtocol {
}
@Override
+ public void renameKey(String volumeName, String bucketName,
+ String fromKeyName, String toKeyName) throws IOException {
+ throw new UnsupportedOperationException("Not yet implemented.");
+ }
+
+ @Override
public List<OzoneKey> listKeys(String volumeName, String bucketName,
String keyPrefix, String prevKey,
int maxListResult)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 2464fe3..ffe93dd 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -520,6 +520,21 @@ public class RpcClient implements ClientProtocol {
}
@Override
+ public void renameKey(String volumeName, String bucketName,
+ String fromKeyName, String toKeyName) throws IOException {
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+ Preconditions.checkNotNull(fromKeyName);
+ Preconditions.checkNotNull(toKeyName);
+ KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(fromKeyName)
+ .build();
+ keySpaceManagerClient.renameKey(keyArgs, toKeyName);
+ }
+
+ @Override
public List<OzoneKey> listKeys(String volumeName, String bucketName,
String keyPrefix, String prevKey,
int maxListResult)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
index 41d523c..678ce92 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
@@ -34,7 +34,7 @@ public final class KsmKeyInfo {
private final String volumeName;
private final String bucketName;
// name of key client specified
- private final String keyName;
+ private String keyName;
private long dataSize;
private List<KsmKeyLocationInfoGroup> keyLocationVersions;
private final long creationTime;
@@ -75,6 +75,10 @@ public final class KsmKeyInfo {
return keyName;
}
+ public void setKeyName(String keyName) {
+ this.keyName = keyName;
+ }
+
public long getDataSize() {
return dataSize;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
index 5da5a27..54862d3 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
@@ -166,12 +166,19 @@ public interface KeySpaceManagerProtocol {
* Look up for the container of an existing key.
*
* @param args the args of the key.
- * @return KsmKeyInfo isntacne that client uses to talk to container.
+ * @return KsmKeyInfo instance that client uses to talk to container.
* @throws IOException
*/
KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
/**
+ * Rename an existing key within a bucket
+ * @param args the args of the key.
+ * @param toKeyName New name to be used for the Key
+ */
+ void renameKey(KsmKeyArgs args, String toKeyName) throws IOException;
+
+ /**
* Deletes an existing key.
*
* @param args the args of the key.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
index cc215cf..854c688 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
@@ -70,6 +70,10 @@ import org.apache.hadoop.ozone.protocol.proto
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.LocateKeyResponse;
import org.apache.hadoop.ozone.protocol.proto
+ .KeySpaceManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto
+ .KeySpaceManagerProtocolProtos.RenameKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
@@ -623,6 +627,29 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo());
}
+ @Override
+ public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
+ RenameKeyRequest.Builder req = RenameKeyRequest.newBuilder();
+ KeyArgs keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(args.getVolumeName())
+ .setBucketName(args.getBucketName())
+ .setKeyName(args.getKeyName())
+ .setDataSize(args.getDataSize()).build();
+ req.setKeyArgs(keyArgs);
+ req.setToKeyName(toKeyName);
+
+ final RenameKeyResponse resp;
+ try {
+ resp = rpcProxy.renameKey(NULL_RPC_CONTROLLER, req.build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ if (resp.getStatus() != Status.OK) {
+ throw new IOException("Rename key failed, error:" +
+ resp.getStatus());
+ }
+ }
+
/**
* Deletes an existing key.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
index 405c5b0..7b70330 100644
--- a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
@@ -50,8 +50,9 @@ enum Status {
BUCKET_ALREADY_EXISTS = 10;
KEY_ALREADY_EXISTS = 11;
KEY_NOT_FOUND = 12;
- ACCESS_DENIED = 13;
- INTERNAL_ERROR = 14;
+ INVALID_KEY_NAME = 13;
+ ACCESS_DENIED = 14;
+ INTERNAL_ERROR = 15;
}
@@ -276,6 +277,15 @@ message SetBucketPropertyResponse {
required Status status = 1;
}
+message RenameKeyRequest{
+ required KeyArgs keyArgs = 1;
+ required string toKeyName = 2;
+}
+
+message RenameKeyResponse{
+ required Status status = 1;
+}
+
message DeleteBucketRequest {
required string volumeName = 1;
required string bucketName = 2;
@@ -413,6 +423,12 @@ service KeySpaceManagerService {
returns(LocateKeyResponse);
/**
+ Rename an existing key within a bucket.
+ */
+ rpc renameKey(RenameKeyRequest)
+ returns(RenameKeyResponse);
+
+ /**
Delete an existing key.
*/
rpc deleteKey(LocateKeyRequest)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index c9a25e5..e3823b3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -528,6 +528,50 @@ public class TestOzoneRpcClient {
}
@Test
+ public void testRenameKey()
+ throws IOException, OzoneException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String fromKeyName = UUID.randomUUID().toString();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ OzoneOutputStream out = bucket.createKey(fromKeyName,
+ value.getBytes().length, ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE);
+ out.write(value.getBytes());
+ out.close();
+ OzoneKey key = bucket.getKey(fromKeyName);
+ Assert.assertEquals(fromKeyName, key.getName());
+
+ // Rename to empty string should fail.
+ IOException ioe = null;
+ String toKeyName = "";
+ try {
+ bucket.renameKey(fromKeyName, toKeyName);
+ } catch (IOException e) {
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+ toKeyName = UUID.randomUUID().toString();
+ bucket.renameKey(fromKeyName, toKeyName);
+
+ // Lookup for old key should fail.
+ try {
+ bucket.getKey(fromKeyName);
+ } catch (IOException e) {
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("Lookup key failed, error"));
+
+ key = bucket.getKey(toKeyName);
+ Assert.assertEquals(toKeyName, key.getName());
+ }
+
+ @Test
public void testListVolume() throws IOException, OzoneException {
String volBase = "vol-" + RandomStringUtils.randomNumeric(3);
//Create 10 volume vol-<random>-a-0-<random> to vol-<random>-a-9-<random>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
index 9a116b7..9c0a488 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
@@ -652,6 +652,121 @@ public class TestKeySpaceManager {
ksmMetrics.getNumKeyDeletesFails());
}
+ /**
+ * Test rename key for ksm.
+ *
+ * @throws IOException
+ * @throws OzoneException
+ */
+ @Test
+ public void testRenameKey() throws IOException, OzoneException {
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ String keyName = "key" + RandomStringUtils.randomNumeric(5);
+ long numKeyRenames = ksmMetrics.getNumKeyRenames();
+ long numKeyRenameFails = ksmMetrics.getNumKeyRenameFails();
+ int testRenameFails = 0;
+ int testRenames = 0;
+ IOException ioe = null;
+
+ VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
+ createVolumeArgs.setUserName(userName);
+ createVolumeArgs.setAdminName(adminName);
+ storageHandler.createVolume(createVolumeArgs);
+
+ BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
+ storageHandler.createBucket(bucketArgs);
+
+ KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
+ keyArgs.setSize(100);
+ String toKeyName = "key" + RandomStringUtils.randomNumeric(5);
+
+ // Rename from non-existent key should fail
+ try {
+ testRenames++;
+ storageHandler.renameKey(keyArgs, toKeyName);
+ } catch (IOException e) {
+ testRenameFails++;
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+ // Write the contents of the key to be renamed
+ String dataString = RandomStringUtils.randomAscii(100);
+ try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
+ stream.write(dataString.getBytes());
+ }
+
+ // Rename the key
+ toKeyName = "key" + RandomStringUtils.randomNumeric(5);
+ testRenames++;
+ storageHandler.renameKey(keyArgs, toKeyName);
+ Assert.assertEquals(numKeyRenames + testRenames,
+ ksmMetrics.getNumKeyRenames());
+ Assert.assertEquals(numKeyRenameFails + testRenameFails,
+ ksmMetrics.getNumKeyRenameFails());
+
+ // Try to get the key, should fail as it has been renamed
+ try {
+ storageHandler.newKeyReader(keyArgs);
+ } catch (IOException e) {
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("KEY_NOT_FOUND"));
+
+ // Verify the contents of the renamed key
+ keyArgs = new KeyArgs(toKeyName, bucketArgs);
+ InputStream in = storageHandler.newKeyReader(keyArgs);
+ byte[] b = new byte[dataString.getBytes().length];
+ in.read(b);
+ Assert.assertEquals(new String(b), dataString);
+
+ // Rewrite the renamed key. Rename to key which already exists should fail.
+ keyArgs = new KeyArgs(keyName, bucketArgs);
+ keyArgs.setSize(100);
+ dataString = RandomStringUtils.randomAscii(100);
+ try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
+ stream.write(dataString.getBytes());
+ stream.close();
+ testRenames++;
+ storageHandler.renameKey(keyArgs, toKeyName);
+ } catch (IOException e) {
+ testRenameFails++;
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+ // Rename to empty string should fail
+ toKeyName = "";
+ try {
+ testRenames++;
+ storageHandler.renameKey(keyArgs, toKeyName);
+ } catch (IOException e) {
+ testRenameFails++;
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+ // Rename from empty string should fail
+ keyArgs = new KeyArgs("", bucketArgs);
+ toKeyName = "key" + RandomStringUtils.randomNumeric(5);
+ try {
+ testRenames++;
+ storageHandler.renameKey(keyArgs, toKeyName);
+ } catch (IOException e) {
+ testRenameFails++;
+ ioe = e;
+ }
+ Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
+
+ Assert.assertEquals(numKeyRenames + testRenames,
+ ksmMetrics.getNumKeyRenames());
+ Assert.assertEquals(numKeyRenameFails + testRenameFails,
+ ksmMetrics.getNumKeyRenameFails());
+ }
+
@Test(timeout = 60000)
public void testListBuckets() throws IOException, OzoneException {
ListBuckets result = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
index 6336c90..338ff63 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
@@ -264,6 +264,15 @@ public interface StorageHandler extends Closeable{
*/
void deleteKey(KeyArgs args) throws IOException, OzoneException;
+ /**
+ * Renames an existing key within a bucket.
+ *
+ * @param args KeyArgs
+ * @param toKeyName New name to be used for the key
+ * @throws OzoneException
+ */
+ void renameKey(KeyArgs args, String toKeyName)
+ throws IOException, OzoneException;
/**
* Returns a list of Key.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
index 9747eac..89158cb 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
@@ -339,6 +339,12 @@ public class LocalStorageHandler implements StorageHandler {
oz.deleteKey(args);
}
+ @Override
+ public void renameKey(KeyArgs args, String toKeyName)
+ throws IOException, OzoneException {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
/**
* Returns a list of Key.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 0f4a856..45270ea 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -457,6 +457,17 @@ public final class DistributedStorageHandler implements StorageHandler {
}
@Override
+ public void renameKey(KeyArgs args, String toKeyName)
+ throws IOException, OzoneException {
+ KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
+ .setVolumeName(args.getVolumeName())
+ .setBucketName(args.getBucketName())
+ .setKeyName(args.getKeyName())
+ .build();
+ keySpaceManagerClient.renameKey(keyArgs, toKeyName);
+ }
+
+ @Override
public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
.setVolumeName(args.getVolumeName())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
index bd29012..8ee67c3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
@@ -52,6 +52,7 @@ public class KSMMetrics {
private @Metric MutableCounterLong numBucketDeletes;
private @Metric MutableCounterLong numKeyAllocate;
private @Metric MutableCounterLong numKeyLookup;
+ private @Metric MutableCounterLong numKeyRenames;
private @Metric MutableCounterLong numKeyDeletes;
private @Metric MutableCounterLong numBucketLists;
private @Metric MutableCounterLong numKeyLists;
@@ -72,6 +73,7 @@ public class KSMMetrics {
private @Metric MutableCounterLong numBucketDeleteFails;
private @Metric MutableCounterLong numKeyAllocateFails;
private @Metric MutableCounterLong numKeyLookupFails;
+ private @Metric MutableCounterLong numKeyRenameFails;
private @Metric MutableCounterLong numKeyDeleteFails;
private @Metric MutableCounterLong numBucketListFails;
private @Metric MutableCounterLong numKeyListFails;
@@ -208,6 +210,16 @@ public class KSMMetrics {
numKeyLookupFails.incr();
}
+ public void incNumKeyRenames() {
+ numKeyOps.incr();
+ numKeyRenames.incr();
+ }
+
+ public void incNumKeyRenameFails() {
+ numKeyOps.incr();
+ numKeyRenameFails.incr();
+ }
+
public void incNumKeyDeleteFails() {
numKeyDeleteFails.incr();
}
@@ -381,6 +393,16 @@ public class KSMMetrics {
}
@VisibleForTesting
+ public long getNumKeyRenames() {
+ return numKeyRenames.value();
+ }
+
+ @VisibleForTesting
+ public long getNumKeyRenameFails() {
+ return numKeyRenameFails.value();
+ }
+
+ @VisibleForTesting
public long getNumKeyDeletes() {
return numKeyDeletes.value();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
index e71ce5f..5ec1db8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
@@ -86,6 +86,16 @@ public interface KeyManager {
KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
/**
+ * Renames an existing key within a bucket.
+ *
+ * @param args the args of the key provided by client.
+ * @param toKeyName New name to be used for the key
+ * @throws IOException if specified key doesn't exist or
+ * some other I/O errors while renaming the key.
+ */
+ void renameKey(KsmKeyArgs args, String toKeyName) throws IOException;
+
+ /**
* Deletes an object by an object key. The key will be immediately removed
* from KSM namespace and become invisible to clients. The object data
* will be removed in async manner that might retain for some time.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
index 8ee7d25..6409a73 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
@@ -396,6 +396,71 @@ public class KeyManagerImpl implements KeyManager {
}
@Override
+ public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
+ Preconditions.checkNotNull(args);
+ Preconditions.checkNotNull(toKeyName);
+ String volumeName = args.getVolumeName();
+ String bucketName = args.getBucketName();
+ String fromKeyName = args.getKeyName();
+ if (toKeyName.length() == 0 || fromKeyName.length() == 0) {
+ LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}.",
+ volumeName, bucketName, fromKeyName, toKeyName);
+ throw new KSMException("Key name is empty",
+ ResultCodes.FAILED_INVALID_KEY_NAME);
+ }
+
+ metadataManager.writeLock().lock();
+ try {
+ // fromKeyName should exist
+ byte[] fromKey = metadataManager.getDBKeyBytes(
+ volumeName, bucketName, fromKeyName);
+ byte[] fromKeyValue = metadataManager.get(fromKey);
+ if (fromKeyValue == null) {
+ // TODO: Add support for renaming open key
+ LOG.error(
+ "Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}. "
+ + "Key: {} not found.", volumeName, bucketName, fromKeyName,
+ toKeyName, fromKeyName);
+ throw new KSMException("Key not found",
+ KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
+ }
+
+ // toKeyName should not exist
+ byte[] toKey =
+ metadataManager.getDBKeyBytes(volumeName, bucketName, toKeyName);
+ byte[] toKeyValue = metadataManager.get(toKey);
+ if (toKeyValue != null) {
+ LOG.error(
+ "Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}. "
+ + "Key: {} already exists.", volumeName, bucketName,
+ fromKeyName, toKeyName, toKeyName);
+ throw new KSMException("Key not found",
+ KSMException.ResultCodes.FAILED_KEY_ALREADY_EXISTS);
+ }
+
+ if (fromKeyName.equals(toKeyName)) {
+ return;
+ }
+
+ KsmKeyInfo newKeyInfo =
+ KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(fromKeyValue));
+ newKeyInfo.setKeyName(toKeyName);
+ newKeyInfo.updateModifcationTime();
+ BatchOperation batch = new BatchOperation();
+ batch.delete(fromKey);
+ batch.put(toKey, newKeyInfo.getProtobuf().toByteArray());
+ metadataManager.writeBatch(batch);
+ } catch (DBException ex) {
+ LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}.",
+ volumeName, bucketName, fromKeyName, toKeyName, ex);
+ throw new KSMException(ex.getMessage(),
+ ResultCodes.FAILED_KEY_RENAME);
+ } finally {
+ metadataManager.writeLock().unlock();
+ }
+ }
+
+ @Override
public void deleteKey(KsmKeyArgs args) throws IOException {
Preconditions.checkNotNull(args);
metadataManager.writeLock().lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
index 120eb06..d0f0c9b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
@@ -744,6 +744,17 @@ public final class KeySpaceManager extends ServiceRuntimeInfoImpl
}
}
+ @Override
+ public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
+ try {
+ metrics.incNumKeyRenames();
+ keyManager.renameKey(args, toKeyName);
+ } catch (IOException e) {
+ metrics.incNumKeyRenameFails();
+ throw e;
+ }
+ }
+
/**
* Deletes an existing key.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
index e2f3580..b902eab 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
@@ -108,6 +108,8 @@ public class KSMException extends IOException {
FAILED_KEY_NOT_FOUND,
FAILED_KEY_ALLOCATION,
FAILED_KEY_DELETION,
+ FAILED_KEY_RENAME,
+ FAILED_INVALID_KEY_NAME,
FAILED_METADATA_ERROR,
FAILED_INTERNAL_ERROR,
KSM_NOT_INITIALIZED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
index 02a4120..536f95a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
@@ -63,6 +63,10 @@ import org.apache.hadoop.ozone.protocol.proto
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.LocateKeyResponse;
import org.apache.hadoop.ozone.protocol.proto
+ .KeySpaceManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto
+ .KeySpaceManagerProtocolProtos.RenameKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
@@ -152,6 +156,8 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
return Status.KEY_ALREADY_EXISTS;
case FAILED_KEY_NOT_FOUND:
return Status.KEY_NOT_FOUND;
+ case FAILED_INVALID_KEY_NAME:
+ return Status.INVALID_KEY_NAME;
default:
return Status.INTERNAL_ERROR;
}
@@ -373,6 +379,26 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
}
@Override
+ public RenameKeyResponse renameKey(
+ RpcController controller, RenameKeyRequest request)
+ throws ServiceException {
+ RenameKeyResponse.Builder resp = RenameKeyResponse.newBuilder();
+ try {
+ KeyArgs keyArgs = request.getKeyArgs();
+ KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
+ .setVolumeName(keyArgs.getVolumeName())
+ .setBucketName(keyArgs.getBucketName())
+ .setKeyName(keyArgs.getKeyName())
+ .build();
+ impl.renameKey(ksmKeyArgs, request.getToKeyName());
+ resp.setStatus(Status.OK);
+ } catch (IOException e){
+ resp.setStatus(exceptionToResponseStatus(e));
+ }
+ return resp.build();
+ }
+
+ @Override
public SetBucketPropertyResponse setBucketProperty(
RpcController controller, SetBucketPropertyRequest request)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/208b97e9/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index c2a2fe2..ef0d3ab 100644
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -257,30 +257,17 @@ public class OzoneFileSystem extends FileSystem {
boolean processKey(String key) throws IOException {
String newKeyName = dstKey.concat(key.substring(srcKey.length()));
- rename(key, newKeyName);
+ bucket.renameKey(key, newKeyName);
return true;
}
-
- // TODO: currently rename work by copying the streams, with changes in KSM,
- // this operation can be improved by renaming the keys in KSM directly.
- private void rename(String src, String dst) throws IOException {
- try (OzoneInputStream inputStream = bucket.readKey(src);
- OzoneOutputStream outputStream = bucket
- .createKey(dst, 0, replicationType, replicationFactor)) {
- IOUtils.copyBytes(inputStream, outputStream, getConf());
- }
- }
}
/**
* Check whether the source and destination path are valid and then perform
- * rename by copying the data from source path to destination path.
+ * rename from source path to destination path.
*
- * The rename operation is performed by copying data from source key
- * to destination key. This is done by reading the source key data into a
- * temporary file and then writing this temporary file to destination key.
- * The temporary file is deleted after the rename operation.
- * TODO: Optimize the operation by renaming keys in KSM.
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
*
* @param src source path for rename
* @param dst destination path for rename
@@ -290,8 +277,11 @@ public class OzoneFileSystem extends FileSystem {
*/
@Override
public boolean rename(Path src, Path dst) throws IOException {
- LOG.trace("rename() from:{} to:{}", src, dst);
+ if (src.equals(dst)) {
+ return true;
+ }
+ LOG.trace("rename() from:{} to:{}", src, dst);
if (src.isRoot()) {
// Cannot rename root of file system
LOG.trace("Cannot rename the root of a filesystem");
@@ -367,8 +357,7 @@ public class OzoneFileSystem extends FileSystem {
}
}
RenameIterator iterator = new RenameIterator(src, dst);
- iterator.iterate();
- return src.equals(dst) || delete(src, true);
+ return iterator.iterate();
}
private class DeleteIterator extends OzoneListingIterator {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org