You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/07/19 21:41:36 UTC
[36/50] hadoop git commit: HADOOP-13254. Create framework for
configurable disk checkers (yufeigu via rkanter)
HADOOP-13254. Create framework for configurable disk checkers (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b105dae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b105dae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b105dae
Branch: refs/heads/YARN-2915
Commit: 8b105daee7a49853e719db7dc98039197dfbb4da
Parents: b390dd6
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Jul 15 13:41:26 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/util/BasicDiskValidator.java | 34 ++++++++
.../org/apache/hadoop/util/DiskValidator.java | 41 +++++++++
.../hadoop/util/DiskValidatorFactory.java | 88 ++++++++++++++++++++
.../hadoop/util/TestBasicDiskValidator.java | 57 +++++++++++++
.../org/apache/hadoop/util/TestDiskChecker.java | 12 +--
.../hadoop/util/TestDiskValidatorFactory.java | 59 +++++++++++++
6 files changed, 285 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java
new file mode 100644
index 0000000..0366687
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * BasicDiskValidator is a wrapper around {@link DiskChecker}.
+ */
+public class BasicDiskValidator implements DiskValidator {
+ public static final String NAME = "basic";
+
+ @Override
+ public void checkStatus(File dir) throws DiskErrorException {
+ DiskChecker.checkDir(dir);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java
new file mode 100644
index 0000000..fa4f589
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * A interface for disk validators.
+ *
+ * The {@link #checkStatus(File)} operation checks status of a file/dir.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface DiskValidator {
+ /**
+ * Check the status of a file/dir.
+ * @param dir a file/dir
+ * @throws DiskErrorException if any disk error
+ */
+ void checkStatus(File dir) throws DiskErrorException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
new file mode 100644
index 0000000..29ab2ad
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The factory class to create instance of {@link DiskValidator}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class DiskValidatorFactory {
+ @VisibleForTesting
+ static final ConcurrentHashMap<Class<? extends DiskValidator>, DiskValidator>
+ INSTANCES = new ConcurrentHashMap<>();
+
+ private DiskValidatorFactory() {
+ }
+
+ /**
+ * Returns a {@link DiskValidator} instance corresponding to the passed clazz.
+ * @param clazz a class extends {@link DiskValidator}
+ */
+ public static DiskValidator
+ getInstance(Class<? extends DiskValidator> clazz) {
+ DiskValidator diskValidator;
+ if (INSTANCES.containsKey(clazz)) {
+ diskValidator = INSTANCES.get(clazz);
+ } else {
+ diskValidator = ReflectionUtils.newInstance(clazz, null);
+ // check the return of putIfAbsent() to see if any other thread have put
+ // the instance with the same key into INSTANCES
+ DiskValidator diskValidatorRet =
+ INSTANCES.putIfAbsent(clazz, diskValidator);
+ if (diskValidatorRet != null) {
+ diskValidator = diskValidatorRet;
+ }
+ }
+
+ return diskValidator;
+ }
+
+ /**
+ * Returns {@link DiskValidator} instance corresponding to its name.
+ * The diskValidator parameter can be "basic" for {@link BasicDiskValidator}.
+ * @param diskValidator canonical class name, for example, "basic"
+ * @throws DiskErrorException if the class cannot be located
+ */
+ @SuppressWarnings("unchecked")
+ public static DiskValidator getInstance(String diskValidator)
+ throws DiskErrorException {
+ @SuppressWarnings("rawtypes")
+ Class clazz;
+
+ if (diskValidator.equalsIgnoreCase(BasicDiskValidator.NAME)) {
+ clazz = BasicDiskValidator.class;
+ } else {
+ try {
+ clazz = Class.forName(diskValidator);
+ } catch (ClassNotFoundException cnfe) {
+ throw new DiskErrorException(diskValidator
+ + " DiskValidator class not found.", cnfe);
+ }
+ }
+
+ return getInstance(clazz);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java
new file mode 100644
index 0000000..c961ee8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * The class to test BasicDiskValidator.
+ */
+public class TestBasicDiskValidator extends TestDiskChecker {
+ @Override
+ protected void checkDirs(boolean isDir, String perm, boolean success)
+ throws Throwable {
+ File localDir = File.createTempFile("test", "tmp");
+ try {
+ if (isDir) {
+ // reuse the file path generated by File#createTempFile to create a dir
+ localDir.delete();
+ localDir.mkdir();
+ }
+
+ Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+ localDir.getAbsolutePath()));
+
+ DiskValidatorFactory.getInstance(BasicDiskValidator.NAME).
+ checkStatus(localDir);
+ assertTrue("call to checkDir() succeeded.", success);
+ } catch (DiskErrorException e) {
+ // call to checkDir() succeeded even though it was expected to fail
+ // if success is false, otherwise throw the exception
+ if (success) {
+ throw e;
+ }
+ } finally {
+ localDir.delete();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
index de54735..b484940 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
@@ -138,30 +138,30 @@ public class TestDiskChecker {
@Test (timeout = 30000)
public void testCheckDir_normal_local() throws Throwable {
- _checkDirs(true, "755", true);
+ checkDirs(true, "755", true);
}
@Test (timeout = 30000)
public void testCheckDir_notDir_local() throws Throwable {
- _checkDirs(false, "000", false);
+ checkDirs(false, "000", false);
}
@Test (timeout = 30000)
public void testCheckDir_notReadable_local() throws Throwable {
- _checkDirs(true, "000", false);
+ checkDirs(true, "000", false);
}
@Test (timeout = 30000)
public void testCheckDir_notWritable_local() throws Throwable {
- _checkDirs(true, "444", false);
+ checkDirs(true, "444", false);
}
@Test (timeout = 30000)
public void testCheckDir_notListable_local() throws Throwable {
- _checkDirs(true, "666", false);
+ checkDirs(true, "666", false);
}
- private void _checkDirs(boolean isDir, String perm, boolean success)
+ protected void checkDirs(boolean isDir, String perm, boolean success)
throws Throwable {
File localDir = File.createTempFile("test", "tmp");
if (isDir) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java
new file mode 100644
index 0000000..1c02b7a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+/**
+ * The class to test DiskValidatorFactory.
+ */
+public class TestDiskValidatorFactory {
+
+ /**
+ * Trivial tests that make sure
+ * {@link DiskValidatorFactory#getInstance(String)} works as expected.
+ *
+ * @throws DiskErrorException if fail to get the instance.
+ */
+ @Test
+ public void testGetInstance() throws DiskErrorException {
+ DiskValidator diskValidator = DiskValidatorFactory.getInstance("basic");
+ assertNotNull("Fail to get the instance.", diskValidator);
+
+ assertEquals("Fail to create the correct instance.",
+ diskValidator.getClass(), BasicDiskValidator.class);
+
+ assertNotNull("Fail to cache the object", DiskValidatorFactory.INSTANCES.
+ get(BasicDiskValidator.class));
+ }
+
+ /**
+ * To test whether an exception is threw out as expected if trying to create
+ * a non-exist class.
+ * @throws DiskErrorException if fail to get the instance.
+ */
+ @Test(expected = DiskErrorException.class)
+ public void testGetInstanceOfNonExistClass() throws DiskErrorException {
+ DiskValidatorFactory.getInstance("non-exist");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org