You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2013/03/26 15:16:15 UTC
svn commit: r1461146 [2/2] - in
/hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/...
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java?rev=1461146&r1=1461145&r2=1461146&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java Tue Mar 26 14:16:13 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.mapred;
+import java.util.Map;
+
import junit.framework.TestCase;
import org.apache.hadoop.mapred.StatisticsCollector.TimeWindow;
@@ -24,6 +26,7 @@ import org.apache.hadoop.mapred.Statisti
public class TestStatisticsCollector extends TestCase{
+ @SuppressWarnings("rawtypes")
public void testMovingWindow() throws Exception {
StatisticsCollector collector = new StatisticsCollector(1);
TimeWindow window = new TimeWindow("test", 6, 2);
@@ -78,6 +81,28 @@ public class TestStatisticsCollector ext
collector.update();
assertEquals((10+10+10+12+13+14), stat.getValues().get(window).getValue());
assertEquals(95, stat.getValues().get(sincStart).getValue());
+
+ // test Stat class
+ Map updaters= collector.getUpdaters();
+ assertEquals(updaters.size(),2);
+ Map<String, Stat> ststistics=collector.getStatistics();
+ assertNotNull(ststistics.get("m1"));
+
+ Stat newStat= collector.createStat("m2");
+ assertEquals(newStat.name, "m2");
+ Stat st=collector.removeStat("m1");
+ assertEquals(st.name, "m1");
+ assertEquals((10+10+10+12+13+14), stat.getValues().get(window).getValue());
+ assertEquals(95, stat.getValues().get(sincStart).getValue());
+ st=collector.removeStat("m1");
+ // try to remove stat again
+ assertNull(st);
+ collector.start();
+ // waiting 2,5 sec
+ Thread.sleep(2500);
+ assertEquals(69, stat.getValues().get(window).getValue());
+ assertEquals(95, stat.getValues().get(sincStart).getValue());
+
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java?rev=1461146&r1=1461145&r2=1461146&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java Tue Mar 26 14:16:13 2013
@@ -61,11 +61,12 @@ public class TestTextInputFormat {
throw new RuntimeException("init failure", e);
}
}
+ @SuppressWarnings("deprecation")
private static Path workDir =
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
"TestTextInputFormat").makeQualified(localFs);
- @Test
+ @Test (timeout=500000)
public void testFormat() throws Exception {
JobConf job = new JobConf(defaultConf);
Path file = new Path(workDir, "test.txt");
@@ -145,7 +146,7 @@ public class TestTextInputFormat {
}
}
- @Test
+ @Test (timeout=900000)
public void testSplitableCodecs() throws IOException {
JobConf conf = new JobConf(defaultConf);
int seed = new Random().nextInt();
@@ -250,7 +251,7 @@ public class TestTextInputFormat {
bufsz);
}
- @Test
+ @Test (timeout=5000)
public void testUTF8() throws Exception {
LineReader in = makeStream("abcd\u20acbdcd\u20ac");
Text line = new Text();
@@ -269,7 +270,7 @@ public class TestTextInputFormat {
*
* @throws Exception
*/
- @Test
+ @Test (timeout=5000)
public void testNewLines() throws Exception {
final String STR = "a\nbb\n\nccc\rdddd\r\r\r\n\r\neeeee";
final int STRLENBYTES = STR.getBytes().length;
@@ -309,7 +310,7 @@ public class TestTextInputFormat {
*
* @throws Exception
*/
- @Test
+ @Test (timeout=5000)
public void testMaxLineLength() throws Exception {
final String STR = "a\nbb\n\nccc\rdddd\r\neeeee";
final int STRLENBYTES = STR.getBytes().length;
@@ -334,7 +335,7 @@ public class TestTextInputFormat {
}
}
- @Test
+ @Test (timeout=5000)
public void testMRMaxLine() throws Exception {
final int MAXPOS = 1024 * 1024;
final int MAXLINE = 10 * 1024;
@@ -354,6 +355,9 @@ public class TestTextInputFormat {
position += b.length;
return b.length;
}
+ public void reset() {
+ position=0;
+ }
};
final LongWritable key = new LongWritable();
final Text val = new Text();
@@ -362,8 +366,14 @@ public class TestTextInputFormat {
conf.setInt(org.apache.hadoop.mapreduce.lib.input.
LineRecordReader.MAX_LINE_LENGTH, MAXLINE);
conf.setInt("io.file.buffer.size", BUF); // used by LRR
- final LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf);
+ // test another constructor
+ LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf);
+ assertFalse("Read a line from null", lrr.next(key, val));
+ infNull.reset();
+ lrr = new LineRecordReader(infNull, 0L, MAXLINE, MAXPOS);
assertFalse("Read a line from null", lrr.next(key, val));
+
+
}
private static void writeFile(FileSystem fs, Path name,
@@ -400,7 +410,7 @@ public class TestTextInputFormat {
/**
* Test using the gzip codec for reading
*/
- @Test
+ @Test (timeout=5000)
public void testGzip() throws IOException {
JobConf job = new JobConf(defaultConf);
CompressionCodec gzip = new GzipCodec();
@@ -434,7 +444,7 @@ public class TestTextInputFormat {
/**
* Test using the gzip codec and an empty input file
*/
- @Test
+ @Test (timeout=5000)
public void testGzipEmpty() throws IOException {
JobConf job = new JobConf(defaultConf);
CompressionCodec gzip = new GzipCodec();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java?rev=1461146&r1=1461145&r2=1461146&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java Tue Mar 26 14:16:13 2013
@@ -44,7 +44,6 @@ public class TestTextOutputFormat extend
"data"),
FileOutputCommitter.TEMP_DIR_NAME), "_" + attempt);
- @SuppressWarnings("unchecked")
public void testFormat() throws Exception {
JobConf job = new JobConf();
job.set(JobContext.TASK_ATTEMPT_ID, attempt);
@@ -59,8 +58,8 @@ public class TestTextOutputFormat extend
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
- TextOutputFormat theOutputFormat = new TextOutputFormat();
- RecordWriter theRecordWriter =
+ TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>();
+ RecordWriter<Object,Object> theRecordWriter =
theOutputFormat.getRecordWriter(localFs, job, file, reporter);
Text key1 = new Text("key1");
@@ -95,7 +94,6 @@ public class TestTextOutputFormat extend
}
- @SuppressWarnings("unchecked")
public void testFormatWithCustomSeparator() throws Exception {
JobConf job = new JobConf();
String separator = "\u0001";
@@ -112,8 +110,8 @@ public class TestTextOutputFormat extend
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
- TextOutputFormat theOutputFormat = new TextOutputFormat();
- RecordWriter theRecordWriter =
+ TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>();
+ RecordWriter<Object,Object> theRecordWriter =
theOutputFormat.getRecordWriter(localFs, job, file, reporter);
Text key1 = new Text("key1");
@@ -147,7 +145,61 @@ public class TestTextOutputFormat extend
assertEquals(output, expectedOutput.toString());
}
-
+ /**
+ * test compressed file
+ * @throws IOException
+ */
+ public void testCompress() throws IOException{
+ JobConf job = new JobConf();
+ String separator = "\u0001";
+ job.set("mapreduce.output.textoutputformat.separator", separator);
+ job.set(JobContext.TASK_ATTEMPT_ID, attempt);
+ job.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS,"true");
+
+ FileOutputFormat.setOutputPath(job, workDir.getParent().getParent());
+ FileOutputFormat.setWorkOutputPath(job, workDir);
+ FileSystem fs = workDir.getFileSystem(job);
+ if (!fs.mkdirs(workDir)) {
+ fail("Failed to create output directory");
+ }
+ String file = "test.txt";
+
+ // A reporter that does nothing
+ Reporter reporter = Reporter.NULL;
+
+ TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>();
+ RecordWriter<Object,Object> theRecordWriter =
+ theOutputFormat.getRecordWriter(localFs, job, file, reporter);
+ Text key1 = new Text("key1");
+ Text key2 = new Text("key2");
+ Text val1 = new Text("val1");
+ Text val2 = new Text("val2");
+ NullWritable nullWritable = NullWritable.get();
+
+ try {
+ theRecordWriter.write(key1, val1);
+ theRecordWriter.write(null, nullWritable);
+ theRecordWriter.write(null, val1);
+ theRecordWriter.write(nullWritable, val2);
+ theRecordWriter.write(key2, nullWritable);
+ theRecordWriter.write(key1, null);
+ theRecordWriter.write(null, null);
+ theRecordWriter.write(key2, val2);
+
+ } finally {
+ theRecordWriter.close(reporter);
+ }
+ File expectedFile = new File(new Path(workDir, file).toString());
+ StringBuffer expectedOutput = new StringBuffer();
+ expectedOutput.append(key1).append(separator).append(val1).append("\n");
+ expectedOutput.append(val1).append("\n");
+ expectedOutput.append(val2).append("\n");
+ expectedOutput.append(key2).append("\n");
+ expectedOutput.append(key1).append("\n");
+ expectedOutput.append(key2).append(separator).append(val2).append("\n");
+ String output = UtilsForTests.slurp(expectedFile);
+ assertEquals(output, expectedOutput.toString());
+ }
public static void main(String[] args) throws Exception {
new TestTextOutputFormat().testFormat();
}