You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/09/06 23:59:23 UTC
hadoop git commit: HDDS-397. Handle deletion for keys with no blocks.
Contributed by Lokesh Jain.
Repository: hadoop
Updated Branches:
refs/heads/trunk 527288ef8 -> 396ce7b88
HDDS-397. Handle deletion for keys with no blocks. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/396ce7b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/396ce7b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/396ce7b8
Branch: refs/heads/trunk
Commit: 396ce7b884d0bf3f85664b2e0e2321203314bb24
Parents: 527288e
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Sep 6 16:58:15 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Sep 6 16:58:15 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 20 +++++++++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 8 ----
.../hadoop/ozone/om/TestKeyDeletingService.java | 47 +++++++++++++++++---
3 files changed, 62 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/396ce7b8/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 06d2587..d21e5c5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.protocol.proto
+ .OzoneManagerProtocolProtos.KeyLocationList;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.BackgroundService;
@@ -447,6 +449,15 @@ public class KeyManagerImpl implements KeyManager {
if (objectValue == null) {
throw new OMException("Key not found",
OMException.ResultCodes.FAILED_KEY_NOT_FOUND);
+ } else {
+ // directly delete key with no blocks from db. This key need not be
+ // moved to deleted table.
+ KeyInfo keyInfo = KeyInfo.parseFrom(objectValue);
+ if (isKeyEmpty(keyInfo)) {
+ metadataManager.getKeyTable().delete(objectKey);
+ LOG.debug("Key {} deleted from OM DB", keyName);
+ return;
+ }
}
metadataManager.getStore().move(objectKey,
metadataManager.getKeyTable(),
@@ -463,6 +474,15 @@ public class KeyManagerImpl implements KeyManager {
}
}
+ private boolean isKeyEmpty(KeyInfo keyInfo) {
+ for (KeyLocationList keyLocationList : keyInfo.getKeyLocationListList()) {
+ if (keyLocationList.getKeyLocationsCount() != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
@Override
public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/396ce7b8/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 16625dc..a1d48ff 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -585,14 +585,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(kv.getValue()));
// Get block keys as a list.
OmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
- if (latest == null) {
- // This means that we have a key without any blocks.
- // BUG-BUG: if this happens the key will never be deleted.
- // TODO: Right thing to do is to remove this key right here.
- LOG.warn("Found a key without blocks: {}, skipping for now.",
- DFSUtil.bytes2String(kv.getKey()));
- continue;
- }
List<BlockID> item = latest.getLocationList().stream()
.map(b -> new BlockID(b.getContainerID(), b.getLocalID()))
.collect(Collectors.toList());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/396ce7b8/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
index 44e3bdf..60c6fc3 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
@@ -33,6 +33,7 @@ import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -91,14 +92,18 @@ public class TestKeyDeletingService {
new ScmBlockLocationTestIngClient(null, null, 0),
metaMgr, conf, UUID.randomUUID().toString());
final int keyCount = 100;
- createAndDeleteKeys(keyManager, keyCount);
+ createAndDeleteKeys(keyManager, keyCount, 1);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
keyManager.start();
+ Assert.assertEquals(
+ keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
GenericTestUtils.waitFor(
() -> keyDeletingService.getDeletedKeyCount().get() >= keyCount,
1000, 10000);
Assert.assertTrue(keyDeletingService.getRunCount().get() > 1);
+ Assert.assertEquals(
+ keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), 0);
}
@Test(timeout = 30000)
@@ -112,22 +117,51 @@ public class TestKeyDeletingService {
new ScmBlockLocationTestIngClient(null, null, 1),
metaMgr, conf, UUID.randomUUID().toString());
final int keyCount = 100;
- createAndDeleteKeys(keyManager, keyCount);
+ createAndDeleteKeys(keyManager, keyCount, 1);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
keyManager.start();
+ Assert.assertEquals(
+ keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
// Make sure that we have run the background thread 5 times more
GenericTestUtils.waitFor(
() -> keyDeletingService.getRunCount().get() >= 5,
100, 1000);
// Since SCM calls are failing, deletedKeyCount should be zero.
Assert.assertEquals(keyDeletingService.getDeletedKeyCount().get(), 0);
+ Assert.assertEquals(
+ keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
+ }
+ @Test(timeout = 30000)
+ public void checkDeletionForEmptyKey()
+ throws IOException, TimeoutException, InterruptedException {
+ OzoneConfiguration conf = createConfAndInitValues();
+ OmMetadataManagerImpl metaMgr = new OmMetadataManagerImpl(conf);
+ //failCallsFrequency = 1 , means all calls fail.
+ KeyManager keyManager =
+ new KeyManagerImpl(
+ new ScmBlockLocationTestIngClient(null, null, 1),
+ metaMgr, conf, UUID.randomUUID().toString());
+ final int keyCount = 100;
+ createAndDeleteKeys(keyManager, keyCount, 0);
+ KeyDeletingService keyDeletingService =
+ (KeyDeletingService) keyManager.getDeletingService();
+ keyManager.start();
+ // Since empty keys are directly deleted from db there should be no
+ // pending deletion keys. Also deletedKeyCount should be zero.
+ Assert.assertEquals(
+ keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), 0);
+ // Make sure that we have run the background thread 2 times or more
+ GenericTestUtils.waitFor(
+ () -> keyDeletingService.getRunCount().get() >= 2,
+ 100, 1000);
+ Assert.assertEquals(keyDeletingService.getDeletedKeyCount().get(), 0);
}
- private void createAndDeleteKeys(KeyManager keyManager, int keyCount)
- throws IOException {
+ private void createAndDeleteKeys(KeyManager keyManager, int keyCount,
+ int numBlocks) throws IOException {
for (int x = 0; x < keyCount; x++) {
String volumeName = String.format("volume%s",
RandomStringUtils.randomAlphanumeric(5));
@@ -153,10 +187,13 @@ public class TestKeyDeletingService {
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
+ .setLocationInfoList(new ArrayList<>())
.build();
//Open, Commit and Delete the Keys in the Key Manager.
OpenKeySession session = keyManager.openKey(arg);
- arg.addLocationInfo(keyManager.allocateBlock(arg, session.getId()));
+ for (int i = 0; i < numBlocks; i++) {
+ arg.addLocationInfo(keyManager.allocateBlock(arg, session.getId()));
+ }
keyManager.commitKey(arg, session.getId());
keyManager.deleteKey(arg);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org