You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/08/01 17:06:30 UTC
[14/50] hadoop git commit: HDDS-270. Move generic container util
functions to ContianerUtils. Contributed by Hanisha Koneru.
HDDS-270. Move generic container util functions to ContianerUtils.
Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cc7ce81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cc7ce81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cc7ce81
Branch: refs/heads/HDFS-12943
Commit: 3cc7ce816e4ffb7287aa05cc5e00b2e058b4a2a4
Parents: 64e739e
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Jul 27 07:12:21 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Jul 27 07:12:21 2018 -0700
----------------------------------------------------------------------
.../common/helpers/ContainerUtils.java | 34 ++++++++++++--------
.../container/common/interfaces/Container.java | 6 ++++
.../container/keyvalue/KeyValueContainer.java | 22 +++++++++----
.../helpers/KeyValueContainerLocationUtil.java | 17 ----------
.../container/ozoneimpl/ContainerReader.java | 27 ++++++----------
.../keyvalue/TestKeyValueContainer.java | 24 ++++----------
.../common/impl/TestContainerPersistence.java | 13 +++-----
7 files changed, 62 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
index 77a891a..469c969 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
@@ -199,20 +199,6 @@ public final class ContainerUtils {
}
/**
- * Returns container file location.
- *
- * @param containerData - Data
- * @param location - Root path
- * @return Path
- */
- public static File getContainerFile(ContainerData containerData,
- Path location) {
- return location.resolve(Long.toString(containerData
- .getContainerID()).concat(CONTAINER_EXTENSION))
- .toFile();
- }
-
- /**
* Persistent a {@link DatanodeDetails} to a local file.
*
* @throws IOException when read/write error occurs
@@ -300,4 +286,24 @@ public final class ContainerUtils {
}
}
+ /**
+ * Get the .container file from the containerBaseDir
+ * @param containerBaseDir container base directory. The name of this
+ * directory is same as the containerID
+ * @return the .container file
+ */
+ public static File getContainerFile(File containerBaseDir) {
+ // Container file layout is
+ // .../<<containerID>>/metadata/<<containerID>>.container
+ String containerFilePath = OzoneConsts.CONTAINER_META_PATH + File.separator
+ + getContainerID(containerBaseDir) + OzoneConsts.CONTAINER_EXTENSION;
+ return new File(containerBaseDir, containerFilePath);
+ }
+
+ /**
+ * ContainerID can be decoded from the container base directory name
+ */
+ public static long getContainerID(File containerBaseDir) {
+ return Long.parseLong(containerBaseDir.getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
index fe35e1d..fc91920 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.util.RwLock;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import java.io.File;
import java.util.Map;
@@ -92,6 +93,11 @@ public interface Container extends RwLock {
ContainerProtos.ContainerType getContainerType();
/**
+ * Returns containerFile.
+ */
+ File getContainerFile();
+
+ /**
* updates the DeleteTransactionId.
* @param deleteTransactionId
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 14f731a..d0e77d2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -114,18 +114,16 @@ public class KeyValueContainer implements Container {
containerMetaDataPath = KeyValueContainerLocationUtil
.getContainerMetaDataPath(hddsVolumeDir, scmId, containerID);
+ containerData.setMetadataPath(containerMetaDataPath.getPath());
+
File chunksPath = KeyValueContainerLocationUtil.getChunksLocationPath(
hddsVolumeDir, scmId, containerID);
- File containerFile = KeyValueContainerLocationUtil.getContainerFile(
- containerMetaDataPath, containerID);
- File dbFile = KeyValueContainerLocationUtil.getContainerDBFile(
- containerMetaDataPath, containerID);
-
// Check if it is new Container.
ContainerUtils.verifyIsNewContainer(containerMetaDataPath);
//Create Metadata path chunks path and metadata db
+ File dbFile = getContainerDBFile();
KeyValueContainerUtil.createContainerMetaData(containerMetaDataPath,
chunksPath, dbFile, config);
@@ -133,13 +131,13 @@ public class KeyValueContainer implements Container {
OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
//Set containerData for the KeyValueContainer.
- containerData.setMetadataPath(containerMetaDataPath.getPath());
containerData.setChunksPath(chunksPath.getPath());
containerData.setContainerDBType(impl);
containerData.setDbFile(dbFile);
containerData.setVolume(containerVolume);
// Create .container file
+ File containerFile = getContainerFile();
writeToContainerFile(containerFile, true);
} catch (StorageContainerException ex) {
@@ -415,12 +413,22 @@ public class KeyValueContainer implements Container {
* Returns containerFile.
* @return .container File name
*/
- private File getContainerFile() {
+ @Override
+ public File getContainerFile() {
return new File(containerData.getMetadataPath(), containerData
.getContainerID() + OzoneConsts.CONTAINER_EXTENSION);
}
/**
+ * Returns container DB file
+ * @return
+ */
+ public File getContainerDBFile() {
+ return new File(containerData.getMetadataPath(), containerData
+ .getContainerID() + OzoneConsts.DN_CONTAINER_DB);
+ }
+
+ /**
* Creates a temporary file.
* @param file
* @return
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java
index 02a8e73..0a81ed8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java
@@ -99,27 +99,10 @@ public final class KeyValueContainerLocationUtil {
}
/**
- * Returns containerFile.
- * @param containerMetaDataPath
- * @param containerID
- * @return .container File name
- */
- public static File getContainerFile(File containerMetaDataPath,
- long containerID) {
- Preconditions.checkNotNull(containerMetaDataPath);
- return new File(containerMetaDataPath, containerID +
- OzoneConsts.CONTAINER_EXTENSION);
- }
-
- /**
* Return containerDB File.
- * @param containerMetaDataPath
- * @param containerID
- * @return containerDB File name
*/
public static File getContainerDBFile(File containerMetaDataPath,
long containerID) {
- Preconditions.checkNotNull(containerMetaDataPath);
return new File(containerMetaDataPath, containerID + OzoneConsts
.DN_CONTAINER_DB);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
index dc33f2e..7c986f0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -32,7 +32,6 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -119,8 +118,7 @@ public class ContainerReader implements Runnable {
}
for (File scmLoc : scmDir) {
- File currentDir = null;
- currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT);
+ File currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT);
File[] containerTopDirs = currentDir.listFiles();
if (containerTopDirs != null) {
for (File containerTopDir : containerTopDirs) {
@@ -128,21 +126,14 @@ public class ContainerReader implements Runnable {
File[] containerDirs = containerTopDir.listFiles();
if (containerDirs != null) {
for (File containerDir : containerDirs) {
- File metadataPath = new File(containerDir + File.separator +
- OzoneConsts.CONTAINER_META_PATH);
- long containerID = Long.parseLong(containerDir.getName());
- if (metadataPath.exists()) {
- File containerFile = KeyValueContainerLocationUtil
- .getContainerFile(metadataPath, containerID);
- if (containerFile.exists()) {
- verifyContainerFile(containerID, containerFile);
- } else {
- LOG.error("Missing .container file for ContainerID: {}",
- containerID);
- }
+ File containerFile = ContainerUtils.getContainerFile(
+ containerDir);
+ long containerID = ContainerUtils.getContainerID(containerDir);
+ if (containerFile.exists()) {
+ verifyContainerFile(containerID, containerFile);
} else {
- LOG.error("Missing container metadata directory for " +
- "ContainerID: {}", containerID);
+ LOG.error("Missing .container file for ContainerID: {}",
+ containerDir.getName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 2bf41e5..35772ff 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -111,11 +111,9 @@ public class TestKeyValueContainer {
File containerMetaDataLoc = new File(containerMetaDataPath);
//Check whether container file and container db file exists or not.
- assertTrue(KeyValueContainerLocationUtil.getContainerFile(
- containerMetaDataLoc, containerID).exists(), ".Container File does" +
- " not exist");
- assertTrue(KeyValueContainerLocationUtil.getContainerDBFile(
- containerMetaDataLoc, containerID).exists(), "Container DB does " +
+ assertTrue(keyValueContainer.getContainerFile().exists(),
+ ".Container File does not exist");
+ assertTrue(keyValueContainer.getContainerDBFile().exists(), "Container DB does " +
"not exist");
}
@@ -166,11 +164,9 @@ public class TestKeyValueContainer {
.getParentFile().exists());
assertFalse("Container File still exists",
- KeyValueContainerLocationUtil.getContainerFile(containerMetaDataLoc,
- containerID).exists());
+ keyValueContainer.getContainerFile().exists());
assertFalse("Container DB file still exists",
- KeyValueContainerLocationUtil.getContainerDBFile(containerMetaDataLoc,
- containerID).exists());
+ keyValueContainer.getContainerDBFile().exists());
}
@@ -188,9 +184,7 @@ public class TestKeyValueContainer {
//Check state in the .container file
String containerMetaDataPath = keyValueContainerData
.getMetadataPath();
- File containerMetaDataLoc = new File(containerMetaDataPath);
- File containerFile = KeyValueContainerLocationUtil.getContainerFile(
- containerMetaDataLoc, containerID);
+ File containerFile = keyValueContainer.getContainerFile();
keyValueContainerData = (KeyValueContainerData) ContainerDataYaml
.readContainerFile(containerFile);
@@ -227,11 +221,7 @@ public class TestKeyValueContainer {
assertEquals(2, keyValueContainerData.getMetadata().size());
//Check metadata in the .container file
- String containerMetaDataPath = keyValueContainerData
- .getMetadataPath();
- File containerMetaDataLoc = new File(containerMetaDataPath);
- File containerFile = KeyValueContainerLocationUtil.getContainerFile(
- containerMetaDataLoc, containerID);
+ File containerFile = keyValueContainer.getContainerFile();
keyValueContainerData = (KeyValueContainerData) ContainerDataYaml
.readContainerFile(containerFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index c2e1645..5322c8e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume
@@ -40,8 +41,6 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.keyvalue.helpers
- .KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
@@ -714,9 +713,7 @@ public class TestContainerPersistence {
KeyValueContainer container =
(KeyValueContainer) addContainer(containerSet, testContainerID);
- File orgContainerFile = KeyValueContainerLocationUtil.getContainerFile(
- new File(container.getContainerData().getMetadataPath()),
- testContainerID);
+ File orgContainerFile = container.getContainerFile();
Assert.assertTrue(orgContainerFile.exists());
Map<String, String> newMetadata = Maps.newHashMap();
@@ -738,9 +735,9 @@ public class TestContainerPersistence {
actualNewData.getMetadata().get("owner"));
// Verify container data on disk
- File newContainerFile = KeyValueContainerLocationUtil.getContainerFile(
- new File(actualNewData.getMetadataPath()),
- testContainerID);
+ File containerBaseDir = new File(actualNewData.getMetadataPath())
+ .getParentFile();
+ File newContainerFile = ContainerUtils.getContainerFile(containerBaseDir);
Assert.assertTrue("Container file should exist.",
newContainerFile.exists());
Assert.assertEquals("Container file should be in same location.",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org