You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/10/12 21:29:31 UTC
hadoop git commit: HDDS-624.PutBlock fails with Unexpected Storage
Container Exception. Contributed by Arpit Agarwal.
Repository: hadoop
Updated Branches:
refs/heads/trunk 8ae8a5004 -> 02e1ef5e0
HDDS-624.PutBlock fails with Unexpected Storage Container Exception.
Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02e1ef5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02e1ef5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02e1ef5e
Branch: refs/heads/trunk
Commit: 02e1ef5e0779369f1363df2bb0437945fe9a271c
Parents: 8ae8a50
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Oct 12 14:22:46 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Oct 12 14:22:46 2018 -0700
----------------------------------------------------------------------
.../hadoop/utils/MetadataStoreBuilder.java | 38 ++++++++++----------
.../hadoop/utils/TestRocksDBStoreMBean.java | 19 ++++++----
.../container/common/utils/ContainerCache.java | 6 ++--
.../container/keyvalue/helpers/BlockUtils.java | 3 +-
4 files changed, 38 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e1ef5e/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
index 85cebed..a1c9b85 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.utils;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.iq80.leveldb.Options;
import org.rocksdb.BlockBasedTableConfig;
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Optional;
+import java.util.function.Supplier;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_METADATA_STORE_IMPL_LEVELDB;
@@ -53,7 +56,7 @@ public class MetadataStoreBuilder {
private File dbFile;
private long cacheSize;
private boolean createIfMissing = true;
- private Configuration conf;
+ private Optional<Configuration> optionalConf = Optional.empty();
private String dbType;
public static MetadataStoreBuilder newBuilder() {
@@ -76,7 +79,7 @@ public class MetadataStoreBuilder {
}
public MetadataStoreBuilder setConf(Configuration configuration) {
- this.conf = configuration;
+ this.optionalConf = Optional.of(configuration);
return this;
}
@@ -98,13 +101,12 @@ public class MetadataStoreBuilder {
}
// Build db store based on configuration
- MetadataStore store = null;
+ final Configuration conf = optionalConf.orElseGet(
+ () -> new OzoneConfiguration());
if(dbType == null) {
LOG.debug("dbType is null, using ");
- dbType = conf == null ?
- OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT :
- conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+ dbType = conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
LOG.debug("dbType is null, using dbType {} from ozone configuration",
dbType);
@@ -117,7 +119,7 @@ public class MetadataStoreBuilder {
if (cacheSize > 0) {
options.cacheSize(cacheSize);
}
- store = new LevelDBStore(dbFile, options);
+ return new LevelDBStore(dbFile, options);
} else if (OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(dbType)) {
org.rocksdb.Options opts = new org.rocksdb.Options();
opts.setCreateIfMissing(createIfMissing);
@@ -128,10 +130,9 @@ public class MetadataStoreBuilder {
opts.setTableFormatConfig(tableConfig);
}
- String rocksDbStat = conf == null ?
- OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT :
- conf.getTrimmed(OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
- OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
+ String rocksDbStat = conf.getTrimmed(
+ OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
+ OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
Statistics statistics = new Statistics();
@@ -139,14 +140,13 @@ public class MetadataStoreBuilder {
opts = opts.setStatistics(statistics);
}
- store = new RocksDBStore(dbFile, opts);
- } else {
- throw new IllegalArgumentException("Invalid argument for "
- + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
- + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
- + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
- + ", but met " + dbType);
+ return new RocksDBStore(dbFile, opts);
}
- return store;
+
+ throw new IllegalArgumentException("Invalid argument for "
+ + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
+ + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
+ + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
+ + ", but met " + dbType);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e1ef5e/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
index a7ce60b..db2572c 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import javax.management.MBeanServer;
@@ -32,15 +33,24 @@ import java.lang.management.ManagementFactory;
* Test the JMX interface for the rocksdb metastore implementation.
*/
public class TestRocksDBStoreMBean {
+
+ Configuration conf;
+
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+ OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
+ }
+
@Test
public void testJmxBeans() throws Exception {
File testDir =
GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat");
- Configuration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
- OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS, "ALL");
RocksDBStore metadataStore =
(RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
@@ -72,9 +82,6 @@ public class TestRocksDBStoreMBean {
File testDir = GenericTestUtils
.getTestDir(getClass().getSimpleName() + "-withoutstat");
- Configuration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
- OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e1ef5e/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
index c63eb73..a533684 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
@@ -123,10 +123,11 @@ public final class ContainerCache extends LRUMap {
* @param containerID - ID of the container.
* @param containerDBType - DB type of the container.
* @param containerDBPath - DB path of the container.
+ * @param conf - Hadoop Configuration.
* @return MetadataStore.
*/
- public MetadataStore getDB(long containerID, String containerDBType, String
- containerDBPath)
+ public MetadataStore getDB(long containerID, String containerDBType,
+ String containerDBPath, Configuration conf)
throws IOException {
Preconditions.checkState(containerID >= 0,
"Container ID cannot be negative.");
@@ -138,6 +139,7 @@ public final class ContainerCache extends LRUMap {
db = MetadataStoreBuilder.newBuilder()
.setDbFile(new File(containerDBPath))
.setCreateIfMissing(false)
+ .setConf(conf)
.setDBType(containerDBType)
.build();
this.put(containerID, db);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e1ef5e/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index b25fec4..d6cadc8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -75,7 +75,8 @@ public final class BlockUtils {
Preconditions.checkNotNull(containerData.getDbFile());
try {
return cache.getDB(containerData.getContainerID(), containerData
- .getContainerDBType(), containerData.getDbFile().getAbsolutePath());
+ .getContainerDBType(), containerData.getDbFile().getAbsolutePath(),
+ conf);
} catch (IOException ex) {
String message = String.format("Error opening DB. Container:%s " +
"ContainerPath:%s", containerData.getContainerID(), containerData
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org