You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by bh...@apache.org on 2020/06/06 04:41:50 UTC
[hadoop-ozone] branch master updated: HDDS-3668. OzoneManager start
fails with RocksDB error on downgrade to older version. (#982)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d57e151 HDDS-3668. OzoneManager start fails with RocksDB error on downgrade to older version. (#982)
d57e151 is described below
commit d57e1514b134aa06f3be4b40f098cd5927ddc081
Author: avijayanhwx <14...@users.noreply.github.com>
AuthorDate: Fri Jun 5 21:41:42 2020 -0700
HDDS-3668. OzoneManager start fails with RocksDB error on downgrade to older version. (#982)
---
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 36 +++++++++++++++++-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 5 +++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 44 ++++++++++++++++++++++
3 files changed, 84 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index 20faf24..4a23541 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -28,6 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.StringUtils;
@@ -41,6 +42,7 @@ import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.DBOptions;
import org.rocksdb.FlushOptions;
+import org.rocksdb.Options;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.TransactionLogIterator;
@@ -94,6 +96,18 @@ public class RDBStore implements DBStore {
this.writeOptions = writeOptions;
try {
+ // This logic has been added to support old column families that have
+ // been removed, or those that may have been created in a future version.
+ // TODO : Revisit this logic during upgrade implementation.
+ List<TableConfig> columnFamiliesInDb = getColumnFamiliesInExistingDb();
+ List<TableConfig> extraCf = columnFamiliesInDb.stream().filter(
+ cf -> !families.contains(cf)).collect(Collectors.toList());
+ if (!extraCf.isEmpty()) {
+ LOG.info("Found the following extra column families in existing DB : " +
+ "{}", extraCf);
+ extraCf.forEach(cf -> columnFamilyDescriptors.add(cf.getDescriptor()));
+ }
+
db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath(),
columnFamilyDescriptors, columnFamilyHandles);
@@ -149,6 +163,26 @@ public class RDBStore implements DBStore {
}
}
+ /**
+ * Read DB and return existing column families.
+ * @return List of column families
+ * @throws RocksDBException on Error.
+ */
+ private List<TableConfig> getColumnFamiliesInExistingDb()
+ throws RocksDBException {
+ List<byte[]> bytes = RocksDB.listColumnFamilies(new Options(),
+ dbLocation.getAbsolutePath());
+ List<TableConfig> columnFamiliesInDb = bytes.stream()
+ .map(cfbytes -> new TableConfig(StringUtils.bytes2String(cfbytes),
+ DBStoreBuilder.HDDS_DEFAULT_DB_PROFILE.getColumnFamilyOptions()))
+ .collect(Collectors.toList());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found column Families in DB : {}",
+ columnFamiliesInDb);
+ }
+ return columnFamiliesInDb;
+ }
+
public static IOException toIOException(String msg, RocksDBException e) {
String statusCode = e.getStatus() == null ? "N/A" :
e.getStatus().getCodeString();
@@ -382,4 +416,4 @@ public class RDBStore implements DBStore {
public RDBMetrics getMetrics() {
return rdbMetrics;
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TableConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TableConfig.java
index fb5c9a6..1bad84e 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TableConfig.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TableConfig.java
@@ -91,4 +91,9 @@ public class TableConfig {
.append(getName())
.toHashCode();
}
+
+ @Override
+ public String toString() {
+ return getName();
+ }
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java
index e162730..d640c4a 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java
@@ -348,5 +348,49 @@ public class TestRDBStore {
}
}
+ @Test
+ public void testDowngrade() throws Exception {
+
+ // Write data to current DB which has 6 column families at the time of
+ // writing this test.
+ for (String family : families) {
+ try (Table table = rdbStore.getTable(family)) {
+ byte[] key = family.getBytes(StandardCharsets.UTF_8);
+ byte[] value =
+ RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8);
+ table.put(key, value);
+ }
+ }
+ // Close current DB.
+ rdbStore.close();
+
+ // Reopen DB with the last column family removed.
+ options = new DBOptions();
+ options.setCreateIfMissing(true);
+ options.setCreateMissingColumnFamilies(true);
+ configSet = new HashSet<>();
+ List<String> familiesMinusOne = families.subList(0, families.size() - 1);
+ for(String name : familiesMinusOne) {
+ TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
+ configSet.add(newConfig);
+ }
+ rdbStore = new RDBStore(rdbStore.getDbLocation(), options, configSet);
+ for (String family : familiesMinusOne) {
+ try (Table table = rdbStore.getTable(family)) {
+ Assert.assertNotNull(family + "is null", table);
+ Object val = table.get(family.getBytes(StandardCharsets.UTF_8));
+ Assert.assertNotNull(val);
+ }
+ }
+
+ // Technically the extra column family should also be open, even though
+ // we do not use it.
+ String extraFamily = families.get(families.size() - 1);
+ try (Table table = rdbStore.getTable(extraFamily)) {
+ Assert.assertNotNull(extraFamily + "is null", table);
+ Object val = table.get(extraFamily.getBytes(StandardCharsets.UTF_8));
+ Assert.assertNotNull(val);
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org