You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/12/03 01:41:35 UTC
[ozone] branch HDDS-3698-upgrade updated: HDDS-4463. Verify that
OM/SCM start fails when Software Layout Version < Metadata Layout Version.
(#1645)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3698-upgrade by this push:
new 9c4f2b4 HDDS-4463. Verify that OM/SCM start fails when Software Layout Version < Metadata Layout Version. (#1645)
9c4f2b4 is described below
commit 9c4f2b4b6ade368a9a28ba87e93fad7fd8f76a5f
Author: Ethan Rose <33...@users.noreply.github.com>
AuthorDate: Wed Dec 2 20:41:23 2020 -0500
HDDS-4463. Verify that OM/SCM start fails when Software Layout Version < Metadata Layout Version. (#1645)
---
.../hadoop/ozone/upgrade/TestUpgradeUtils.java | 51 ++++++++++++++
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 75 ++++++++++++++++++++
.../hadoop/hdds/scm/upgrade/package-info.java | 21 ++++++
.../hadoop/ozone/TestStorageContainerManager.java | 54 ++++++++-------
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 80 ++++++++++++++++++++++
5 files changed, 256 insertions(+), 25 deletions(-)
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeUtils.java
new file mode 100644
index 0000000..e494488
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeUtils.java
@@ -0,0 +1,51 @@
+/*
+ package org.apache.hadoop.ozone.upgrade;
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.upgrade;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.common.StorageInfo;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+public final class TestUpgradeUtils {
+ private TestUpgradeUtils() { }
+
+ /**
+ * Creates a VERSION file for the specified node type under the directory
+ * {@code parentDir}.
+ */
+ public static File createVersionFile(File parentDir,
+ HddsProtos.NodeType nodeType, int mlv) throws IOException {
+
+ final String versionFileName = "VERSION";
+
+ StorageInfo info = new StorageInfo(
+ nodeType,
+ UUID.randomUUID().toString(),
+ System.currentTimeMillis(),
+ mlv);
+
+ File versionFile = new File(parentDir, versionFileName);
+ info.writeTo(versionFile);
+
+ return versionFile;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java
new file mode 100644
index 0000000..73dc612
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.upgrade;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeatureCatalog;
+import org.apache.hadoop.ozone.upgrade.LayoutFeature;
+import org.apache.hadoop.ozone.upgrade.TestUpgradeUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Tests that SCM will throw an exception on creation when it reads in a
+ * VERSION file indicating a metadata layout version larger than its
+ * software layout version.
+ */
+public class TestScmStartupSlvLessThanMlv {
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Test
+ public void testStartupSlvLessThanMlv() throws Exception {
+ // Add subdirectories under the temporary folder where the version file
+ // will be placed.
+ File scmSubdir = tempFolder.newFolder("scm", "current");
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS,
+ tempFolder.getRoot().getAbsolutePath());
+
+ // Set metadata layout version larger then software layout version.
+ int largestSlv = 0;
+ for (LayoutFeature f: HDDSLayoutFeatureCatalog.HDDSLayoutFeature.values()) {
+ largestSlv = Math.max(largestSlv, f.layoutVersion());
+ }
+ int mlv = largestSlv + 1;
+
+ // Create version file with MLV > SLV, which should fail the SCM
+ // construction.
+ TestUpgradeUtils.createVersionFile(scmSubdir, HddsProtos.NodeType.SCM, mlv);
+
+ try {
+ new StorageContainerManager(conf);
+ Assert.fail("Expected IOException due to incorrect MLV on SCM creation.");
+ } catch(IOException e) {
+ String expectedMessage = String.format("Metadata layout version (%s) > " +
+ "software layout version (%s)", mlv, largestSlv);
+ GenericTestUtils.assertExceptionContains(expectedMessage, e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/package-info.java
new file mode 100644
index 0000000..19fe9ec
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.upgrade;
+/**
+ * Tests related to SCM upgrade.
+ */
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 4b68f7e..035602c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -17,6 +17,35 @@
*/
package org.apache.hadoop.ozone;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic
+ .NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+ .HDDS_COMMAND_STATUS_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+ .HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+ .HDDS_SCM_SAFEMODE_PIPELINE_CREATION;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
@@ -72,34 +101,12 @@ import org.junit.Test;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.time.Duration;
-import java.util.Map;
-import java.util.List;
-import java.util.Set;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY;
-import static org.apache.hadoop.hdds.HddsConfigKeys.*;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.*;
/**
* Test class that exercises the StorageContainerManager.
@@ -121,9 +128,6 @@ public class TestStorageContainerManager {
@Rule
public ExpectedException exception = ExpectedException.none();
- @Rule
- public TemporaryFolder folder= new TemporaryFolder();
-
@BeforeClass
public static void setup() throws IOException {
xceiverClientManager = new XceiverClientManager(new OzoneConfiguration());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupSlvLessThanMlv.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupSlvLessThanMlv.java
new file mode 100644
index 0000000..db8afad
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupSlvLessThanMlv.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import java.io.File;
+import java.util.UUID;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.upgrade.LayoutFeature;
+import org.apache.hadoop.ozone.upgrade.TestUpgradeUtils;
+
+
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test that the ozone manager will not start when it loads a VERSION file
+ * indicating a metadata layout version larger than its software layout version.
+ */
+public class TestOmStartupSlvLessThanMlv {
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Test
+ public void testStartupSlvLessThanMlv() throws Exception {
+ // Add subdirectories under the temporary folder where the version file
+ // will be placed.
+ File omSubdir = tempFolder.newFolder("om", "current");
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_DB_DIRS,
+ tempFolder.getRoot().getAbsolutePath());
+
+ // Set metadata layout version larger then software layout version.
+ int largestSlv = 0;
+ for (LayoutFeature f: OMLayoutFeature.values()) {
+ largestSlv = Math.max(largestSlv, f.layoutVersion());
+ }
+ int mlv = largestSlv + 1;
+
+ // Create version file with MLV > SLV, which should fail the cluster build.
+ TestUpgradeUtils.createVersionFile(omSubdir, HddsProtos.NodeType.OM, mlv);
+
+ MiniOzoneCluster.Builder clusterBuilder = MiniOzoneCluster.newBuilder(conf)
+ .setClusterId(UUID.randomUUID().toString())
+ .setScmId(UUID.randomUUID().toString())
+ .setOmId(UUID.randomUUID().toString());
+
+ try {
+ clusterBuilder.build();
+ Assert.fail("Expected OMException due to incorrect MLV on OM creation.");
+ } catch(OMException e) {
+ String expectedMessage = String.format("Cannot initialize " +
+ "VersionManager. Metadata layout version (%s) > software layout" +
+ " version (%s)", mlv, largestSlv);
+ GenericTestUtils.assertExceptionContains(expectedMessage, e);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org