You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by br...@apache.org on 2013/03/24 19:25:14 UTC
git commit: MRUNIT-162: MapReduce testing utilties
Updated Branches:
refs/heads/trunk 19ef2d3a7 -> 30e2d1fde
MRUNIT-162: MapReduce testing utilties
Project: http://git-wip-us.apache.org/repos/asf/mrunit/repo
Commit: http://git-wip-us.apache.org/repos/asf/mrunit/commit/30e2d1fd
Tree: http://git-wip-us.apache.org/repos/asf/mrunit/tree/30e2d1fd
Diff: http://git-wip-us.apache.org/repos/asf/mrunit/diff/30e2d1fd
Branch: refs/heads/trunk
Commit: 30e2d1fde029a6071ed4643d819aa758d723085a
Parents: 19ef2d3
Author: Brock Noland <br...@apache.org>
Authored: Sun Mar 24 13:24:15 2013 -0500
Committer: Brock Noland <br...@apache.org>
Committed: Sun Mar 24 13:24:15 2013 -0500
----------------------------------------------------------------------
pom.xml | 6 +
.../hadoop/mrunit/testutil/TemporaryPath.java | 179 +++++++++++++++
.../hadoop/mrunit/testutil/TestTemporaryPath.java | 100 ++++++++
3 files changed, 285 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mrunit/blob/30e2d1fd/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9549d72..4fdfb7d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -347,6 +347,12 @@
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>11.0.2</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${specificHadoopVersion}</version>
http://git-wip-us.apache.org/repos/asf/mrunit/blob/30e2d1fd/src/main/java/org/apache/hadoop/mrunit/testutil/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/testutil/TemporaryPath.java b/src/main/java/org/apache/hadoop/mrunit/testutil/TemporaryPath.java
new file mode 100644
index 0000000..1b8ca5c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/mrunit/testutil/TemporaryPath.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.testutil;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.Rule;
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+
+/**
+ * A JUnit {@link Rule} that creates a managed temporary directory for a test case and
+ * destroys it afterwards. This works much like JUnit's {@link TemporaryFolder} but is
+ * geared towards Hadoop applications. The temporary directory is deleted after each test
+ * case, no matter if the test case succeeded or failed.
+ *
+ * <p>This is how it works:</p>
+ *
+ * <pre><tt>
+ * public class TestExample {
+ * {@literal @Rule}
+ * public TemporaryPath tmpDir = new TemporaryPath();
+ *
+ * {@literal @Test}
+ * public void testSomething() {
+ * Path input = tmpDir.copyResourcePath("my-test-data");
+ * Path output = tmpDir.getPath("output");
+ *
+ * // create and run a Hadoop job reading from input and writing to output
+ * }
+ * }
+ * </tt></pre>
+ *
+ * <p>In some cases, the frameworks you use in your tests use temporary directories
+ * internally. If those directories are configurable via Hadoop properties, you can
+ * let {@link TemporaryPath#overridePathProperties(Configuration)} override them
+ * so that they point to your managed temporary directory. You have to specify the
+ * properties to override via the constructor.</p>
+ */
+public final class TemporaryPath extends ExternalResource {
+ private final TemporaryFolder tmp = new TemporaryFolder();
+ private final Set<String> confKeys;
+
+ /**
+ * Construct {@link TemporaryPath}.
+ * @param confKeys {@link Configuration} keys containing directories to override
+ */
+ public TemporaryPath(String... confKeys) {
+ if (confKeys != null) {
+ this.confKeys = ImmutableSet.copyOf(confKeys);
+ } else {
+ this.confKeys = ImmutableSet.of();
+ }
+ }
+
+ /**
+ * Used by JUnit internally.
+ */
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return tmp.apply(base, description);
+ }
+
+ /**
+ * Get the root directory which will be deleted automatically.
+ */
+ public File getRootFile() {
+ return tmp.getRoot();
+ }
+
+ /**
+ * Get the root directory as a {@link Path}.
+ */
+ public Path getRootPath() {
+ return toPath(tmp.getRoot());
+ }
+
+ /**
+ * Get the root directory as an absolute file name.
+ */
+ public String getRootFileName() {
+ return tmp.getRoot().getAbsolutePath();
+ }
+
+ /**
+ * Get a {@link File} below the temporary directory.
+ */
+ public File getFile(String fileName) {
+ return new File(getRootFile(), fileName);
+ }
+
+ /**
+ * Get a {@link Path} below the temporary directory.
+ */
+ public Path getPath(String fileName) {
+ return toPath(getFile(fileName));
+ }
+
+ /**
+ * Get an absolute file name below the temporary directory.
+ */
+ public String getFileName(String fileName) {
+ return getFile(fileName).getAbsolutePath();
+ }
+
+ /**
+ * Copy a classpath resource to {@link File}.
+ */
+ public File copyResourceFile(String resourceName) throws IOException {
+ File dest = new File(tmp.getRoot(), resourceName);
+ copy(resourceName, dest);
+ return dest;
+ }
+
+ /**
+ * Copy a classpath resource to a {@link Path}.
+ */
+ public Path copyResourcePath(String resourceName) throws IOException {
+ return toPath(copyResourceFile(resourceName));
+ }
+
+
+ /**
+ * Get a new {@link Configuration} instance.
+ */
+ public Configuration getDefaultConfiguration() {
+ return overridePathProperties(new Configuration());
+ }
+
+ /**
+ * Set all keys specified in the constructor to temporary directories.
+ */
+ public Configuration overridePathProperties(Configuration conf) {
+ for (String name : confKeys) {
+ conf.set(name, getFileName("tmp-" + name));
+ }
+ return conf;
+ }
+
+ /**
+ * Copy a classpath resource returning its absolute file name.
+ */
+ public String copyResourceFileName(String resourceName) throws IOException {
+ return copyResourceFile(resourceName).getAbsolutePath();
+ }
+
+ private void copy(String resourceName, File dest) throws IOException {
+ Files.copy(Resources.newInputStreamSupplier(Resources.getResource(resourceName)), dest);
+ }
+
+ private static Path toPath(File file) {
+ return new Path(file.getAbsolutePath());
+ }
+}
http://git-wip-us.apache.org/repos/asf/mrunit/blob/30e2d1fd/src/test/java/org/apache/hadoop/mrunit/testutil/TestTemporaryPath.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/mrunit/testutil/TestTemporaryPath.java b/src/test/java/org/apache/hadoop/mrunit/testutil/TestTemporaryPath.java
new file mode 100644
index 0000000..5a5e2de
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/mrunit/testutil/TestTemporaryPath.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.testutil;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.junit.Rule;
+import org.junit.Test;
+
+
+public class TestTemporaryPath {
+ @Rule
+ public TemporaryPath tmpDir = new TemporaryPath("foo.tmp", "bar.tmp");
+
+ @Test
+ public void testRoot() throws IOException {
+ assertThat(tmpDir.getRootFile().exists(), is(true));
+ assertThat(new File(tmpDir.getRootFileName()).exists(), is(true));
+ assertThat(getFs().exists(tmpDir.getRootPath()), is(true));
+ }
+
+ @Test
+ public void testFile() throws IOException {
+ assertThat(tmpDir.getFile("foo").getParentFile(), is(tmpDir.getRootFile()));
+ assertThat(tmpDir.getFile("foo").getName(), is("foo"));
+ assertThat(tmpDir.getFile("foo").exists(), is(false));
+ }
+
+ @Test
+ public void testPath() throws IOException {
+ assertThat(tmpDir.getPath("foo").getParent(), is(tmpDir.getRootPath()));
+ assertThat(tmpDir.getPath("foo").getName(), is("foo"));
+ assertThat(getFs().exists(tmpDir.getPath("foo")), is(false));
+ }
+
+ @Test
+ public void testFileName() {
+ assertThat(new File(tmpDir.getRootFileName()), is(tmpDir.getRootFile()));
+ assertThat(new File(tmpDir.getFileName("foo").toString()), is(tmpDir.getFile("foo")));
+ }
+
+ @Test
+ public void testCopyResource() throws IOException {
+ File dest = tmpDir.getFile("testfile");
+ assertThat(dest.exists(), is(false));
+
+ tmpDir.copyResourceFile("testfile");
+ assertThat(dest.exists(), is(true));
+ }
+
+ @Test
+ public void testGetDefaultConfiguration() {
+ Configuration conf = tmpDir.getDefaultConfiguration();
+ String fooDir = conf.get("foo.tmp");
+ String barDir = conf.get("bar.tmp");
+
+ assertThat(fooDir.startsWith(tmpDir.getRootFileName()), is(true));
+ assertThat(barDir.startsWith(tmpDir.getRootFileName()), is(true));
+ assertThat(fooDir, is(not(barDir)));
+ }
+
+ @Test
+ public void testOverridePathProperties() {
+ Configuration conf = new Configuration();
+ conf.set("foo.tmp", "whatever");
+ conf.set("other.dir", "/my/dir");
+
+ tmpDir.overridePathProperties(conf);
+
+ assertThat(conf.get("foo.tmp").startsWith(tmpDir.getRootFileName()), is(true));
+ assertThat(conf.get("other.dir"), is("/my/dir"));
+ }
+
+ private LocalFileSystem getFs() throws IOException {
+ return FileSystem.getLocal(new Configuration());
+ }
+}