You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by je...@apache.org on 2013/12/02 20:04:51 UTC
svn commit: r1547149 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/ap...
Author: jeagles
Date: Mon Dec 2 19:04:51 2013
New Revision: 1547149
URL: http://svn.apache.org/r1547149
Log:
MAPREDUCE-5640. Rename TestLineRecordReader in jobclient module (Jason Lowe via jeagles)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java
Removed:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1547149&r1=1547148&r2=1547149&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Dec 2 19:04:51 2013
@@ -1512,6 +1512,9 @@ Release 0.23.10 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-5640. Rename TestLineRecordReader in jobclient module (Jason Lowe
+ via jeagles)
+
OPTIMIZATIONS
MAPREDUCE-1981. Improve getSplits performance by using listLocatedStatus
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java?rev=1547149&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java Mon Dec 2 19:04:51 2013
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.lib.IdentityMapper;
+import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.junit.Test;
+
+public class TestLineRecordReaderJobs {
+
+ private static Path workDir = new Path(new Path(System.getProperty(
+ "test.build.data", "."), "data"), "TestTextInputFormat");
+ private static Path inputDir = new Path(workDir, "input");
+ private static Path outputDir = new Path(workDir, "output");
+
+ /**
+ * Writes the input test file
+ *
+ * @param conf
+ * @throws IOException
+ */
+ public void createInputFile(Configuration conf) throws IOException {
+ FileSystem localFs = FileSystem.getLocal(conf);
+ Path file = new Path(inputDir, "test.txt");
+ Writer writer = new OutputStreamWriter(localFs.create(file));
+ writer.write("abc\ndef\t\nghi\njkl");
+ writer.close();
+ }
+
+ /**
+ * Reads the output file into a string
+ *
+ * @param conf
+ * @return
+ * @throws IOException
+ */
+ public String readOutputFile(Configuration conf) throws IOException {
+ FileSystem localFs = FileSystem.getLocal(conf);
+ Path file = new Path(outputDir, "part-00000");
+ return UtilsForTests.slurpHadoop(file, localFs);
+ }
+
+ /**
+ * Creates and runs an MR job
+ *
+ * @param conf
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
+ */
+ public void createAndRunJob(Configuration conf) throws IOException,
+ InterruptedException, ClassNotFoundException {
+ JobConf job = new JobConf(conf);
+ job.setJarByClass(TestLineRecordReaderJobs.class);
+ job.setMapperClass(IdentityMapper.class);
+ job.setReducerClass(IdentityReducer.class);
+ FileInputFormat.addInputPath(job, inputDir);
+ FileOutputFormat.setOutputPath(job, outputDir);
+ JobClient.runJob(job);
+ }
+
+ /**
+ * Test the case when a custom record delimiter is specified using the
+ * textinputformat.record.delimiter configuration property
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
+ */
+ @Test
+ public void testCustomRecordDelimiters() throws IOException,
+ InterruptedException, ClassNotFoundException {
+ Configuration conf = new Configuration();
+ conf.set("textinputformat.record.delimiter", "\t\n");
+ conf.setInt("mapreduce.job.maps", 1);
+ FileSystem localFs = FileSystem.getLocal(conf);
+ // cleanup
+ localFs.delete(workDir, true);
+ // creating input test file
+ createInputFile(conf);
+ createAndRunJob(conf);
+ String expected = "0\tabc\ndef\n9\tghi\njkl\n";
+ assertEquals(expected, readOutputFile(conf));
+ }
+
+ /**
+ * Test the default behavior when the textinputformat.record.delimiter
+ * configuration property is not specified
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
+ */
+ @Test
+ public void testDefaultRecordDelimiters() throws IOException,
+ InterruptedException, ClassNotFoundException {
+ Configuration conf = new Configuration();
+ FileSystem localFs = FileSystem.getLocal(conf);
+ // cleanup
+ localFs.delete(workDir, true);
+ // creating input test file
+ createInputFile(conf);
+ createAndRunJob(conf);
+ String expected = "0\tabc\n4\tdef\t\n9\tghi\n13\tjkl\n";
+ assertEquals(expected, readOutputFile(conf));
+ }
+
+}
\ No newline at end of file
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java?rev=1547149&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java Mon Dec 2 19:04:51 2013
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.input;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.UtilsForTests;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.Test;
+
+public class TestLineRecordReaderJobs {
+
+ private static Path workDir = new Path(new Path(System.getProperty(
+ "test.build.data", "."), "data"), "TestTextInputFormat");
+ private static Path inputDir = new Path(workDir, "input");
+ private static Path outputDir = new Path(workDir, "output");
+
+ /**
+ * Writes the input test file
+ *
+ * @param conf
+ * @throws IOException
+ */
+ public void createInputFile(Configuration conf) throws IOException {
+ FileSystem localFs = FileSystem.getLocal(conf);
+ Path file = new Path(inputDir, "test.txt");
+ Writer writer = new OutputStreamWriter(localFs.create(file));
+ writer.write("abc\ndef\t\nghi\njkl");
+ writer.close();
+ }
+
+ /**
+ * Reads the output file into a string
+ *
+ * @param conf
+ * @return
+ * @throws IOException
+ */
+ public String readOutputFile(Configuration conf) throws IOException {
+ FileSystem localFs = FileSystem.getLocal(conf);
+ Path file = new Path(outputDir, "part-r-00000");
+ return UtilsForTests.slurpHadoop(file, localFs);
+ }
+
+ /**
+ * Creates and runs an MR job
+ *
+ * @param conf
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
+ */
+ public void createAndRunJob(Configuration conf) throws IOException,
+ InterruptedException, ClassNotFoundException {
+ Job job = Job.getInstance(conf);
+ job.setJarByClass(TestLineRecordReaderJobs.class);
+ job.setMapperClass(Mapper.class);
+ job.setReducerClass(Reducer.class);
+ FileInputFormat.addInputPath(job, inputDir);
+ FileOutputFormat.setOutputPath(job, outputDir);
+ job.waitForCompletion(true);
+ }
+
+ /**
+ * Test the case when a custom record delimiter is specified using the
+ * textinputformat.record.delimiter configuration property
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
+ */
+ @Test
+ public void testCustomRecordDelimiters() throws IOException,
+ InterruptedException, ClassNotFoundException {
+ Configuration conf = new Configuration();
+ conf.set("textinputformat.record.delimiter", "\t\n");
+ FileSystem localFs = FileSystem.getLocal(conf);
+ // cleanup
+ localFs.delete(workDir, true);
+ // creating input test file
+ createInputFile(conf);
+ createAndRunJob(conf);
+ String expected = "0\tabc\ndef\n9\tghi\njkl\n";
+ assertEquals(expected, readOutputFile(conf));
+ }
+
+ /**
+ * Test the default behavior when the textinputformat.record.delimiter
+ * configuration property is not specified
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
+ */
+ @Test
+ public void testDefaultRecordDelimiters() throws IOException,
+ InterruptedException, ClassNotFoundException {
+ Configuration conf = new Configuration();
+ FileSystem localFs = FileSystem.getLocal(conf);
+ // cleanup
+ localFs.delete(workDir, true);
+ // creating input test file
+ createInputFile(conf);
+ createAndRunJob(conf);
+ String expected = "0\tabc\n4\tdef\t\n9\tghi\n13\tjkl\n";
+ assertEquals(expected, readOutputFile(conf));
+ }
+
+}