You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/04/15 17:55:20 UTC
[hbase] branch branch-2 updated: HBASE-24175 [Flakey Tests]
TestSecureExportSnapshot FileNotFoundException Addendum: add test to check
for '/tmp' references.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 62c5bdc HBASE-24175 [Flakey Tests] TestSecureExportSnapshot FileNotFoundException Addendum: add test to check for '/tmp' references.
62c5bdc is described below
commit 62c5bdcc0898b337b55b7a6d8c092e7903526ecc
Author: stack <st...@apache.org>
AuthorDate: Wed Apr 15 10:37:55 2020 -0700
HBASE-24175 [Flakey Tests] TestSecureExportSnapshot FileNotFoundException
Addendum: add test to check for '/tmp' references.
---
.../hbase/snapshot/TestExportSnapshotAdjunct.java | 17 +++++++
.../apache/hadoop/hbase/HBaseTestingUtility.java | 57 ++++++++++++++--------
2 files changed, 53 insertions(+), 21 deletions(-)
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
index f1be09c..a61b572 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hbase.snapshot;
+import static org.junit.Assert.assertFalse;
+import java.util.Iterator;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -68,6 +71,20 @@ public class TestExportSnapshotAdjunct {
TestExportSnapshot.setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
+ Configuration conf = TEST_UTIL.getConfiguration();
+ for (Iterator<Map.Entry<String, String>> i = conf.iterator(); i.hasNext();) {
+ Map.Entry<String, String> e = i.next();
+ if (e.getValue().contains("java.io.tmpdir")) {
+ continue;
+ }
+ if (e.getValue().contains("hadoop.tmp.dir")) {
+ continue;
+ }
+ if (e.getValue().contains("hbase.tmp.dir")) {
+ continue;
+ }
+ assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("tmp"));
+ }
}
@AfterClass
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index d4c3c13..14040d9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -456,21 +456,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
createSubDir(
"mapreduce.cluster.local.dir",
testPath, "mapred-local-dir");
-
- // Frustrate yarn's attempts at writing /tmp.
- String property = "yarn.node-labels.fs-store.root-dir";
- createSubDir(property, testPath, property);
- property = "yarn.nodemanager.log-dirs";
- createSubDir(property, testPath, property);
- property = "yarn.nodemanager.remote-app-log-dir";
- createSubDir(property, testPath, property);
- property = "yarn.timeline-service.entity-group-fs-store.active-dir";
- createSubDir(property, testPath, property);
- property = "yarn.timeline-service.entity-group-fs-store.done-dir";
- createSubDir(property, testPath, property);
- property = "yarn.nodemanager.remote-app-log-dir";
- createSubDir(property, testPath, property);
-
return testPath;
}
@@ -713,16 +698,24 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
return dfsCluster;
}
- /** This is used before starting HDFS and map-reduce mini-clusters */
+ /** This is used before starting HDFS and map-reduce mini-clusters
+ * Run something like the below to check for the likes of '/tmp' references -- i.e.
+ * references outside of the test data dir -- in the conf.
+ * Configuration conf = TEST_UTIL.getConfiguration();
+ * for (Iterator<Map.Entry<String, String>> i = conf.iterator(); i.hasNext();) {
+ * Map.Entry<String, String> e = i.next();
+ * assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("/tmp"));
+ * }
+ */
private void createDirsAndSetProperties() throws IOException {
setupClusterTestDir();
conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
- createDirAndSetProperty("cache_data", "test.cache.data");
- createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
- hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
- createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir");
- createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir");
+ createDirAndSetProperty("test.cache.data", "test.cache.data");
+ createDirAndSetProperty("hadoop.tmp.dir", "hadoop.tmp.dir");
+ hadoopLogDir = createDirAndSetProperty("hadoop.log.dir", "hadoop.log.dir");
+ createDirAndSetProperty("mapreduce.cluster.local.dir", "mapreduce.cluster.local.dir");
+ createDirAndSetProperty("mapreduce.cluster.temp.dir", "mapreduce.cluster.temp.dir");
enableShortCircuit();
Path root = getDataTestDirOnTestFS("hadoop");
@@ -734,6 +727,28 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString());
conf.set("yarn.app.mapreduce.am.staging-dir",
new Path(root, "mapreduce-am-staging-root-dir").toString());
+
+ // Frustrate yarn's and hdfs's attempts at writing /tmp.
+ String property = "yarn.node-labels.fs-store.root-dir";
+ createDirAndSetProperty(property, property);
+ property = "yarn.nodemanager.log-dirs";
+ createDirAndSetProperty(property, property);
+ property = "yarn.nodemanager.remote-app-log-dir";
+ createDirAndSetProperty(property, property);
+ property = "yarn.timeline-service.entity-group-fs-store.active-dir";
+ createDirAndSetProperty(property, property);
+ property = "yarn.timeline-service.entity-group-fs-store.done-dir";
+ createDirAndSetProperty(property, property);
+ property = "yarn.nodemanager.remote-app-log-dir";
+ createDirAndSetProperty(property, property);
+ property = "dfs.journalnode.edits.dir";
+ createDirAndSetProperty(property, property);
+ property = "dfs.datanode.shared.file.descriptor.paths";
+ createDirAndSetProperty(property, property);
+ property = "nfs.dump.dir";
+ createDirAndSetProperty(property, property);
+ property = "java.io.tmpdir";
+ createDirAndSetProperty(property, property);
}
/**