You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by hu...@apache.org on 2020/03/25 16:58:21 UTC
[hbase] branch branch-1 updated: HBASE-23957 [flakey test]
client.TestMultiParallel fails to read hbase-site.xml (#1310) (#1344)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new 61f9048 HBASE-23957 [flakey test] client.TestMultiParallel fails to read hbase-site.xml (#1310) (#1344)
61f9048 is described below
commit 61f9048b019a2c45d799cb238a74d7766e578ec4
Author: Huaxiang Sun <qi...@hotmail.com>
AuthorDate: Wed Mar 25 09:58:04 2020 -0700
HBASE-23957 [flakey test] client.TestMultiParallel fails to read hbase-site.xml (#1310) (#1344)
Signed-off-by: Nick Dimiduk <nd...@apache.org>
Signed-off-by: stack <st...@apache.org>
---
.../client/AbstractTestUpdateConfiguration.java | 116 +++++++++++++++++++++
.../hbase/client/TestUpdateConfiguration.java | 48 ++++-----
.../{hbase-site2.xml => override-hbase-site.xml} | 0
3 files changed, 134 insertions(+), 30 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestUpdateConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestUpdateConfiguration.java
new file mode 100644
index 0000000..ec6204d
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestUpdateConfiguration.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+
+/**
+ * Base class to test Configuration Update logic. It wraps up things needed to
+ * test configuration change and provides utility methods for test cluster setup,
+ * updating/restoring configuration file.
+ */
+public abstract class AbstractTestUpdateConfiguration {
+ private static final String SERVER_CONFIG = "hbase-site.xml";
+ private static final String OVERRIDE_SERVER_CONFIG = "override-hbase-site.xml";
+ private static final String BACKUP_SERVER_CONFIG = "backup-hbase-site.xml";
+
+ private static Path configFileUnderTestDataDir;
+ private static Path overrideConfigFileUnderTestDataDir;
+ private static Path backupConfigFileUnderTestDataDir;
+
+ protected static void setUpConfigurationFiles(final HBaseTestingUtility testUtil)
+ throws Exception {
+ // Before this change, the test will update hbase-site.xml under target/test-classes and
+ // trigger a config reload. Since target/test-classes/hbase-site.xml is being used by
+ // other testing cases at the same time, this update will break other testing cases so it will
+ // be flakey in nature.
+ // To avoid this, the change is to make target/test-classes/hbase-site.xml immutable. A new
+ // hbase-site.xml will be created under its test data directory, i.e,
+ // hbase-server/target/test-data/UUID, this new file will be added as a resource for the
+ // config, new update will be applied to this new file and only visible to this specific test
+ // case. The target/test-classes/hbase-site.xml will not be changed during the test.
+
+ String absoluteDataPath = testUtil.getDataTestDir().toString();
+
+ // Create test-data directories.
+ Files.createDirectories(Paths.get(absoluteDataPath));
+
+ // Copy hbase-site.xml from target/test-class to target/test-data/UUID directory.
+ Path configFile = Paths.get("target", "test-classes", SERVER_CONFIG);
+ configFileUnderTestDataDir = Paths.get(absoluteDataPath, SERVER_CONFIG);
+ Files.copy(configFile, configFileUnderTestDataDir);
+
+ // Copy override config file overrider-hbase-site.xml from target/test-class to
+ // target/test-data/UUID directory.
+ Path overrideConfigFile = Paths.get("target", "test-classes",
+ OVERRIDE_SERVER_CONFIG);
+ overrideConfigFileUnderTestDataDir = Paths.get(absoluteDataPath, OVERRIDE_SERVER_CONFIG);
+ Files.copy(overrideConfigFile, overrideConfigFileUnderTestDataDir);
+
+ backupConfigFileUnderTestDataDir = Paths.get(absoluteDataPath, BACKUP_SERVER_CONFIG);
+
+ // Add the new custom config file to Configuration
+ testUtil.getConfiguration().addResource(testUtil.getDataTestDir(SERVER_CONFIG));
+ }
+
+ protected static void addResourceToRegionServerConfiguration(final HBaseTestingUtility testUtil) {
+ // When RegionServer is created in MiniHBaseCluster, it uses HBaseConfiguration.create(conf) of
+ // the master Configuration. The create() just copies config params over, it does not do
+ // a clone for a historic reason. Properties such as resources are lost during this process.
+ // Exposing a new method in HBaseConfiguration causes confusion. Instead, the new hbase-site.xml
+ // under test-data directory is added to RegionServer's configuration as a workaround.
+ for (RegionServerThread rsThread : testUtil.getMiniHBaseCluster().getRegionServerThreads()) {
+ rsThread.getRegionServer().getConfiguration().addResource(
+ testUtil.getDataTestDir(SERVER_CONFIG));
+ }
+ }
+
+ /**
+ * Replace the hbase-site.xml file under this test's data directory with the content of the
+ * override-hbase-site.xml file. Stashes the current existing file so that it can be restored
+ * using {@link #restoreHBaseSiteXML()}.
+ *
+ * @throws IOException if an I/O error occurs
+ */
+ protected void replaceHBaseSiteXML() throws IOException {
+ // make a backup of hbase-site.xml
+ Files.copy(configFileUnderTestDataDir,
+ backupConfigFileUnderTestDataDir, StandardCopyOption.REPLACE_EXISTING);
+ // update hbase-site.xml by overwriting it
+ Files.copy(overrideConfigFileUnderTestDataDir,
+ configFileUnderTestDataDir, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ /**
+ * Restores the hbase-site.xml file that was stashed by a previous call to
+ * {@link #replaceHBaseSiteXML()}.
+ *
+ * @throws IOException if an I/O error occurs
+ */
+ protected void restoreHBaseSiteXML() throws IOException {
+ // restore hbase-site.xml
+ Files.copy(backupConfigFileUnderTestDataDir,
+ configFileUnderTestDataDir, StandardCopyOption.REPLACE_EXISTING);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
index 226c9e7..bf2da18 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
@@ -21,11 +21,6 @@ package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -33,22 +28,29 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
@Category({MediumTests.class})
-public class TestUpdateConfiguration {
+public class TestUpdateConfiguration extends AbstractTestUpdateConfiguration {
private static final Log LOG = LogFactory.getLog(TestUpdateConfiguration.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@BeforeClass
public static void setup() throws Exception {
+ setUpConfigurationFiles(TEST_UTIL);
TEST_UTIL.startMiniCluster(2, 1);
+ addResourceToRegionServerConfiguration(TEST_UTIL);
}
+ @Rule
+ public TestName name = new TestName();
+
@Test
public void testOnlineConfigChange() throws IOException {
- LOG.debug("Starting the test");
+ LOG.debug("Starting the test " + name.getMethodName());
Admin admin = TEST_UTIL.getHBaseAdmin();
ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
admin.updateConfiguration(server);
@@ -56,42 +58,29 @@ public class TestUpdateConfiguration {
@Test
public void testMasterOnlineConfigChange() throws IOException {
- LOG.debug("Starting the test");
- Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml");
- Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml");
- Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml");
- // make a backup of hbase-site.xml
- Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING);
- // update hbase-site.xml by overwriting it
- Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
-
+ LOG.debug("Starting the test " + name.getMethodName());
+ replaceHBaseSiteXML();
Admin admin = TEST_UTIL.getHBaseAdmin();
+
ServerName server = TEST_UTIL.getHBaseCluster().getMaster().getServerName();
admin.updateConfiguration(server);
Configuration conf = TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
int custom = conf.getInt("hbase.custom.config", 0);
- assertEquals(custom, 1000);
- // restore hbase-site.xml
- Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+ assertEquals(1000, custom);
+ restoreHBaseSiteXML();
}
@Test
public void testAllOnlineConfigChange() throws IOException {
- LOG.debug("Starting the test");
+ LOG.debug("Starting the test " + name.getMethodName());
Admin admin = TEST_UTIL.getHBaseAdmin();
admin.updateConfiguration();
}
@Test
public void testAllCustomOnlineConfigChange() throws IOException {
- LOG.debug("Starting the test");
- Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml");
- Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml");
- Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml");
- // make a backup of hbase-site.xml
- Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING);
- // update hbase-site.xml by overwriting it
- Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+ LOG.debug("Starting the test " + name.getMethodName());
+ replaceHBaseSiteXML();
Admin admin = TEST_UTIL.getHBaseAdmin();
admin.updateConfiguration();
@@ -109,7 +98,6 @@ public class TestUpdateConfiguration {
custom = regionServerConfiguration.getInt("hbase.custom.config", 0);
assertEquals(custom, 1000);
- // restore hbase-site.xml
- Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+ restoreHBaseSiteXML();
}
}
diff --git a/hbase-server/src/test/resources/hbase-site2.xml b/hbase-server/src/test/resources/override-hbase-site.xml
similarity index 100%
rename from hbase-server/src/test/resources/hbase-site2.xml
rename to hbase-server/src/test/resources/override-hbase-site.xml