You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2012/07/19 21:06:22 UTC
svn commit: r1363482 [2/2] - in /hadoop/common/branches/branch-1: ./
src/contrib/streaming/src/test/org/apache/hadoop/streaming/
src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/io/compress/
src/core/org/apache/hadoop/io/compress/bzip2/ src/ma...
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java?rev=1363482&r1=1363481&r2=1363482&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java Thu Jul 19 19:06:21 2012
@@ -20,16 +20,20 @@ package org.apache.hadoop.mapred;
import java.io.*;
import java.util.*;
-import junit.framework.TestCase;
-import org.apache.commons.logging.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.ReflectionUtils;
-public class TestTextInputFormat extends TestCase {
+import org.junit.Test;
+import static junit.framework.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TestTextInputFormat {
private static final Log LOG =
LogFactory.getLog(TestTextInputFormat.class.getName());
@@ -39,17 +43,20 @@ public class TestTextInputFormat extends
private static FileSystem localFs = null;
static {
try {
+ defaultConf.set("fs.default.name", "file:///");
localFs = FileSystem.getLocal(defaultConf);
} catch (IOException e) {
throw new RuntimeException("init failure", e);
}
}
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "."), "data"),
- "TestTextInputFormat");
-
+
+ private static Path workDir =
+ new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+ "TestTextInputFormat").makeQualified(localFs);
+
+ @Test
public void testFormat() throws Exception {
- JobConf job = new JobConf();
+ JobConf job = new JobConf(defaultConf);
Path file = new Path(workDir, "test.txt");
// A reporter that does nothing
@@ -127,6 +134,95 @@ public class TestTextInputFormat extends
}
}
+ @Test
+ public void testSplitableCodecs() throws IOException {
+ JobConf conf = new JobConf(defaultConf);
+ int seed = new Random().nextInt();
+ // Create the codec
+ CompressionCodec codec = null;
+ try {
+ codec = (CompressionCodec)
+ ReflectionUtils.newInstance(conf.getClassByName("org.apache.hadoop.io.compress.BZip2Codec"), conf);
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException("Illegal codec!");
+ }
+ Path file = new Path(workDir, "test"+codec.getDefaultExtension());
+
+ // A reporter that does nothing
+ Reporter reporter = Reporter.NULL;
+ LOG.info("seed = "+seed);
+ Random random = new Random(seed);
+ FileSystem localFs = FileSystem.getLocal(conf);
+
+ localFs.delete(workDir, true);
+ FileInputFormat.setInputPaths(conf, workDir);
+
+ final int MAX_LENGTH = 500000;
+
+ // for a variety of lengths
+ for (int length = MAX_LENGTH / 2; length < MAX_LENGTH;
+ length += random.nextInt(MAX_LENGTH / 4)+1) {
+
+ LOG.info("creating; entries = " + length);
+
+ // create a file with length entries
+ Writer writer =
+ new OutputStreamWriter(codec.createOutputStream(localFs.create(file)));
+ try {
+ for (int i = 0; i < length; i++) {
+ writer.write(Integer.toString(i));
+ writer.write("\n");
+ }
+ } finally {
+ writer.close();
+ }
+
+ // try splitting the file in a variety of sizes
+ TextInputFormat format = new TextInputFormat();
+ format.configure(conf);
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ for (int i = 0; i < 3; i++) {
+ int numSplits = random.nextInt(MAX_LENGTH/2000)+1;
+ LOG.info("splitting: requesting = " + numSplits);
+ InputSplit[] splits = format.getSplits(conf, numSplits);
+ LOG.info("splitting: got = " + splits.length);
+
+ // check each split
+ BitSet bits = new BitSet(length);
+ for (int j = 0; j < splits.length; j++) {
+ LOG.debug("split["+j+"]= " + splits[j]);
+ RecordReader<LongWritable, Text> reader =
+ format.getRecordReader(splits[j], conf, reporter);
+ try {
+ int counter = 0;
+ while (reader.next(key, value)) {
+ int v = Integer.parseInt(value.toString());
+ LOG.debug("read " + v);
+
+ if (bits.get(v)) {
+ LOG.warn("conflict with " + v +
+ " in split " + j +
+ " at position "+reader.getPos());
+ }
+ assertFalse("Key in multiple partitions.", bits.get(v));
+ bits.set(v);
+ counter++;
+ }
+ if (counter > 0) {
+ LOG.info("splits["+j+"]="+splits[j]+" count=" + counter);
+ } else {
+ LOG.debug("splits["+j+"]="+splits[j]+" count=" + counter);
+ }
+ } finally {
+ reader.close();
+ }
+ }
+ assertEquals("Some keys in no partition.", length, bits.cardinality());
+ }
+ }
+ }
+
private static LineReader makeStream(String str) throws IOException {
return new LineReader(new ByteArrayInputStream
(str.getBytes("UTF-8")),
@@ -138,6 +234,7 @@ public class TestTextInputFormat extends
bufsz);
}
+ @Test
public void testUTF8() throws Exception {
LineReader in = makeStream("abcd\u20acbdcd\u20ac");
Text line = new Text();
@@ -156,6 +253,7 @@ public class TestTextInputFormat extends
*
* @throws Exception
*/
+ @Test
public void testNewLines() throws Exception {
final String STR = "a\nbb\n\nccc\rdddd\r\r\r\n\r\neeeee";
final int STRLENBYTES = STR.getBytes().length;
@@ -195,6 +293,7 @@ public class TestTextInputFormat extends
*
* @throws Exception
*/
+ @Test
public void testMaxLineLength() throws Exception {
final String STR = "a\nbb\n\nccc\rdddd\r\neeeee";
final int STRLENBYTES = STR.getBytes().length;
@@ -253,8 +352,9 @@ public class TestTextInputFormat extends
/**
* Test using the gzip codec for reading
*/
- public static void testGzip() throws IOException {
- JobConf job = new JobConf();
+ @Test
+ public void testGzip() throws IOException {
+ JobConf job = new JobConf(defaultConf);
CompressionCodec gzip = new GzipCodec();
ReflectionUtils.setConf(gzip, job);
localFs.delete(workDir, true);
@@ -286,8 +386,9 @@ public class TestTextInputFormat extends
/**
* Test using the gzip codec and an empty input file
*/
- public static void testGzipEmpty() throws IOException {
- JobConf job = new JobConf();
+ @Test
+ public void testGzipEmpty() throws IOException {
+ JobConf job = new JobConf(defaultConf);
CompressionCodec gzip = new GzipCodec();
ReflectionUtils.setConf(gzip, job);
localFs.delete(workDir, true);
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestMapReduceLocal.java?rev=1363482&r1=1363481&r2=1363482&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestMapReduceLocal.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestMapReduceLocal.java Thu Jul 19 19:06:21 2012
@@ -97,7 +97,7 @@ public class TestMapReduceLocal extends
private float last = 0.0f;
private boolean progressCalled = false;
@Override
- public float getProgress() {
+ public float getProgress() throws IOException {
progressCalled = true;
final float ret = super.getProgress();
assertTrue("getProgress decreased", ret >= last);