You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2014/04/11 06:51:23 UTC
svn commit: r1586573 - in
/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/io/ src/test/java/org/apache/hadoop/io/
Author: cnauroth
Date: Fri Apr 11 04:51:23 2014
New Revision: 1586573
URL: http://svn.apache.org/r1586573
Log:
HADOOP-10490. Merging change r1586572 from branch-2 to branch-2.4
Modified:
hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java
hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java
Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1586573&r1=1586572&r2=1586573&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/CHANGES.txt Fri Apr 11 04:51:23 2014
@@ -24,6 +24,9 @@ Release 2.4.1 - UNRELEASED
HADOOP-10473. TestCallQueueManager should interrupt before counting calls.
(szetszwo)
+ HADOOP-10490. TestMapFile and TestBloomMapFile leak file descriptors.
+ (cnauroth)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java?rev=1586573&r1=1586572&r2=1586573&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java Fri Apr 11 04:51:23 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.util.Options;
@@ -836,21 +837,24 @@ public class MapFile {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.getLocal(conf);
- MapFile.Reader reader = new MapFile.Reader(fs, in, conf);
- MapFile.Writer writer =
- new MapFile.Writer(conf, fs, out,
- reader.getKeyClass().asSubclass(WritableComparable.class),
- reader.getValueClass());
-
- WritableComparable key =
- ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf);
- Writable value =
- ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf);
-
- while (reader.next(key, value)) // copy all entries
- writer.append(key, value);
-
- writer.close();
+ MapFile.Reader reader = null;
+ MapFile.Writer writer = null;
+ try {
+ reader = new MapFile.Reader(fs, in, conf);
+ writer =
+ new MapFile.Writer(conf, fs, out,
+ reader.getKeyClass().asSubclass(WritableComparable.class),
+ reader.getValueClass());
+
+ WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass()
+ .asSubclass(WritableComparable.class), conf);
+ Writable value = ReflectionUtils.newInstance(reader.getValueClass()
+ .asSubclass(Writable.class), conf);
+
+ while (reader.next(key, value)) // copy all entries
+ writer.append(key, value);
+ } finally {
+ IOUtils.cleanup(LOG, writer, reader);
+ }
}
-
}
Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java?rev=1586573&r1=1586572&r2=1586573&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java Fri Apr 11 04:51:23 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.io;
+import static org.mockito.Mockito.*;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,6 +33,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -63,39 +66,44 @@ public class TestBloomMapFile extends Te
FileSystem fs = FileSystem.getLocal(conf);
Path qualifiedDirName = fs.makeQualified(TEST_DIR);
conf.setInt("io.mapfile.bloom.size", 2048);
- BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs,
- qualifiedDirName.toString(), IntWritable.class, Text.class);
- IntWritable key = new IntWritable();
- Text value = new Text();
- for (int i = 0; i < 2000; i += 2) {
- key.set(i);
- value.set("00" + i);
- writer.append(key, value);
- }
- writer.close();
-
- BloomMapFile.Reader reader = new BloomMapFile.Reader(fs,
- qualifiedDirName.toString(), conf);
- // check false positives rate
- int falsePos = 0;
- int falseNeg = 0;
- for (int i = 0; i < 2000; i++) {
- key.set(i);
- boolean exists = reader.probablyHasKey(key);
- if (i % 2 == 0) {
- if (!exists)
- falseNeg++;
- } else {
- if (exists)
- falsePos++;
+ BloomMapFile.Writer writer = null;
+ BloomMapFile.Reader reader = null;
+ try {
+ writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(),
+ IntWritable.class, Text.class);
+ IntWritable key = new IntWritable();
+ Text value = new Text();
+ for (int i = 0; i < 2000; i += 2) {
+ key.set(i);
+ value.set("00" + i);
+ writer.append(key, value);
+ }
+ writer.close();
+
+ reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf);
+ // check false positives rate
+ int falsePos = 0;
+ int falseNeg = 0;
+ for (int i = 0; i < 2000; i++) {
+ key.set(i);
+ boolean exists = reader.probablyHasKey(key);
+ if (i % 2 == 0) {
+ if (!exists)
+ falseNeg++;
+ } else {
+ if (exists)
+ falsePos++;
+ }
}
+ reader.close();
+ fs.delete(qualifiedDirName, true);
+ System.out.println("False negatives: " + falseNeg);
+ assertEquals(0, falseNeg);
+ System.out.println("False positives: " + falsePos);
+ assertTrue(falsePos < 2);
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
- reader.close();
- fs.delete(qualifiedDirName, true);
- System.out.println("False negatives: " + falseNeg);
- assertEquals(0, falseNeg);
- System.out.println("False positives: " + falsePos);
- assertTrue(falsePos < 2);
}
@SuppressWarnings("deprecation")
@@ -103,23 +111,28 @@ public class TestBloomMapFile extends Te
throws Exception {
FileSystem fs = FileSystem.getLocal(conf);
Path qualifiedDirName = fs.makeQualified(TEST_DIR);
- BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs,
- qualifiedDirName.toString(), Text.class, NullWritable.class);
- for (Text key : keys) {
- writer.append(key, NullWritable.get());
- }
- writer.close();
-
- // will check for membership in the opposite order of how keys were inserted
- BloomMapFile.Reader reader = new BloomMapFile.Reader(fs,
- qualifiedDirName.toString(), conf);
- Collections.reverse(keys);
- for (Text key : keys) {
- assertTrue("False negative for existing key " + key,
+ BloomMapFile.Writer writer = null;
+ BloomMapFile.Reader reader = null;
+ try {
+ writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(),
+ Text.class, NullWritable.class);
+ for (Text key : keys) {
+ writer.append(key, NullWritable.get());
+ }
+ writer.close();
+
+ // will check for membership in opposite order of how keys were inserted
+ reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf);
+ Collections.reverse(keys);
+ for (Text key : keys) {
+ assertTrue("False negative for existing key " + key,
reader.probablyHasKey(key));
+ }
+ reader.close();
+ fs.delete(qualifiedDirName, true);
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
- reader.close();
- fs.delete(qualifiedDirName, true);
}
public void testMembershipVaryingSizedKeysTest1() throws Exception {
@@ -140,15 +153,19 @@ public class TestBloomMapFile extends Te
* test {@code BloomMapFile.delete()} method
*/
public void testDeleteFile() {
+ BloomMapFile.Writer writer = null;
try {
FileSystem fs = FileSystem.getLocal(conf);
- BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
+ writer = new BloomMapFile.Writer(conf, TEST_FILE,
MapFile.Writer.keyClass(IntWritable.class),
MapFile.Writer.valueClass(Text.class));
assertNotNull("testDeleteFile error !!!", writer);
- BloomMapFile.delete(fs, "." + TEST_FILE);
+ writer.close();
+ BloomMapFile.delete(fs, TEST_FILE.toString());
} catch (Exception ex) {
fail("unexpect ex in testDeleteFile !!!");
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -157,24 +174,26 @@ public class TestBloomMapFile extends Te
* IOException
*/
public void testIOExceptionInWriterConstructor() {
- Path dirNameSpy = org.mockito.Mockito.spy(TEST_FILE);
+ Path dirNameSpy = spy(TEST_FILE);
+ BloomMapFile.Reader reader = null;
+ BloomMapFile.Writer writer = null;
try {
- BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
+ writer = new BloomMapFile.Writer(conf, TEST_FILE,
MapFile.Writer.keyClass(IntWritable.class),
MapFile.Writer.valueClass(Text.class));
writer.append(new IntWritable(1), new Text("123124142"));
writer.close();
- org.mockito.Mockito.when(dirNameSpy.getFileSystem(conf)).thenThrow(
- new IOException());
- BloomMapFile.Reader reader = new BloomMapFile.Reader(dirNameSpy, conf,
+ when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException());
+ reader = new BloomMapFile.Reader(dirNameSpy, conf,
MapFile.Reader.comparator(new WritableComparator(IntWritable.class)));
assertNull("testIOExceptionInWriterConstructor error !!!",
reader.getBloomFilter());
- reader.close();
} catch (Exception ex) {
fail("unexpect ex in testIOExceptionInWriterConstructor !!!");
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -183,8 +202,10 @@ public class TestBloomMapFile extends Te
*/
public void testGetBloomMapFile() {
int SIZE = 10;
+ BloomMapFile.Reader reader = null;
+ BloomMapFile.Writer writer = null;
try {
- BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
+ writer = new BloomMapFile.Writer(conf, TEST_FILE,
MapFile.Writer.keyClass(IntWritable.class),
MapFile.Writer.valueClass(Text.class));
@@ -193,7 +214,7 @@ public class TestBloomMapFile extends Te
}
writer.close();
- BloomMapFile.Reader reader = new BloomMapFile.Reader(TEST_FILE, conf,
+ reader = new BloomMapFile.Reader(TEST_FILE, conf,
MapFile.Reader.comparator(new WritableComparator(IntWritable.class)));
for (int i = 0; i < SIZE; i++) {
@@ -203,9 +224,10 @@ public class TestBloomMapFile extends Te
assertNull("testGetBloomMapFile error !!!",
reader.get(new IntWritable(SIZE + 5), new Text()));
- reader.close();
} catch (Exception ex) {
fail("unexpect ex in testGetBloomMapFile !!!");
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -214,36 +236,46 @@ public class TestBloomMapFile extends Te
*/
@SuppressWarnings("deprecation")
public void testBloomMapFileConstructors() {
+ BloomMapFile.Writer writer = null;
try {
FileSystem ts = FileSystem.get(conf);
String testFileName = TEST_FILE.toString();
- BloomMapFile.Writer writer1 = new BloomMapFile.Writer(conf, ts,
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, IntWritable.class, Text.class, CompressionType.BLOCK,
defaultCodec, defaultProgress);
- assertNotNull("testBloomMapFileConstructors error !!!", writer1);
- BloomMapFile.Writer writer2 = new BloomMapFile.Writer(conf, ts,
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, IntWritable.class, Text.class, CompressionType.BLOCK,
defaultProgress);
- assertNotNull("testBloomMapFileConstructors error !!!", writer2);
- BloomMapFile.Writer writer3 = new BloomMapFile.Writer(conf, ts,
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, IntWritable.class, Text.class, CompressionType.BLOCK);
- assertNotNull("testBloomMapFileConstructors error !!!", writer3);
- BloomMapFile.Writer writer4 = new BloomMapFile.Writer(conf, ts,
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, IntWritable.class, Text.class, CompressionType.RECORD,
defaultCodec, defaultProgress);
- assertNotNull("testBloomMapFileConstructors error !!!", writer4);
- BloomMapFile.Writer writer5 = new BloomMapFile.Writer(conf, ts,
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, IntWritable.class, Text.class, CompressionType.RECORD,
defaultProgress);
- assertNotNull("testBloomMapFileConstructors error !!!", writer5);
- BloomMapFile.Writer writer6 = new BloomMapFile.Writer(conf, ts,
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, IntWritable.class, Text.class, CompressionType.RECORD);
- assertNotNull("testBloomMapFileConstructors error !!!", writer6);
- BloomMapFile.Writer writer7 = new BloomMapFile.Writer(conf, ts,
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
+ writer = new BloomMapFile.Writer(conf, ts,
testFileName, WritableComparator.get(Text.class), Text.class);
- assertNotNull("testBloomMapFileConstructors error !!!", writer7);
+ assertNotNull("testBloomMapFileConstructors error !!!", writer);
+ writer.close();
} catch (Exception ex) {
fail("testBloomMapFileConstructors error !!!");
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -272,13 +304,13 @@ public class TestBloomMapFile extends Te
@Override
public CompressionOutputStream createOutputStream(OutputStream out,
Compressor compressor) throws IOException {
- return null;
+ return mock(CompressionOutputStream.class);
}
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
throws IOException {
- return null;
+ return mock(CompressionOutputStream.class);
}
@Override
Modified: hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java?rev=1586573&r1=1586572&r2=1586573&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java Fri Apr 11 04:51:23 2014
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -68,13 +69,13 @@ public class TestMapFile {
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
throws IOException {
- return null;
+ return mock(CompressionOutputStream.class);
}
@Override
public CompressionOutputStream createOutputStream(OutputStream out,
Compressor compressor) throws IOException {
- return null;
+ return mock(CompressionOutputStream.class);
}
@Override
@@ -138,46 +139,52 @@ public class TestMapFile {
@Test
public void testGetClosestOnCurrentApi() throws Exception {
final String TEST_PREFIX = "testGetClosestOnCurrentApi.mapfile";
- MapFile.Writer writer = createWriter(TEST_PREFIX, Text.class, Text.class);
- int FIRST_KEY = 1;
- // Test keys: 11,21,31,...,91
- for (int i = FIRST_KEY; i < 100; i += 10) {
- Text t = new Text(Integer.toString(i));
- writer.append(t, t);
- }
- writer.close();
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
+ try {
+ writer = createWriter(TEST_PREFIX, Text.class, Text.class);
+ int FIRST_KEY = 1;
+ // Test keys: 11,21,31,...,91
+ for (int i = FIRST_KEY; i < 100; i += 10) {
+ Text t = new Text(Integer.toString(i));
+ writer.append(t, t);
+ }
+ writer.close();
- MapFile.Reader reader = createReader(TEST_PREFIX, Text.class);
- Text key = new Text("55");
- Text value = new Text();
-
- // Test get closest with step forward
- Text closest = (Text) reader.getClosest(key, value);
- assertEquals(new Text("61"), closest);
-
- // Test get closest with step back
- closest = (Text) reader.getClosest(key, value, true);
- assertEquals(new Text("51"), closest);
-
- // Test get closest when we pass explicit key
- final Text explicitKey = new Text("21");
- closest = (Text) reader.getClosest(explicitKey, value);
- assertEquals(new Text("21"), explicitKey);
-
- // Test what happens at boundaries. Assert if searching a key that is
- // less than first key in the mapfile, that the first key is returned.
- key = new Text("00");
- closest = (Text) reader.getClosest(key, value);
- assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
-
- // Assert that null is returned if key is > last entry in mapfile.
- key = new Text("92");
- closest = (Text) reader.getClosest(key, value);
- assertNull("Not null key in testGetClosestWithNewCode", closest);
-
- // If we were looking for the key before, we should get the last key
- closest = (Text) reader.getClosest(key, value, true);
- assertEquals(new Text("91"), closest);
+ reader = createReader(TEST_PREFIX, Text.class);
+ Text key = new Text("55");
+ Text value = new Text();
+
+ // Test get closest with step forward
+ Text closest = (Text) reader.getClosest(key, value);
+ assertEquals(new Text("61"), closest);
+
+ // Test get closest with step back
+ closest = (Text) reader.getClosest(key, value, true);
+ assertEquals(new Text("51"), closest);
+
+ // Test get closest when we pass explicit key
+ final Text explicitKey = new Text("21");
+ closest = (Text) reader.getClosest(explicitKey, value);
+ assertEquals(new Text("21"), explicitKey);
+
+ // Test what happens at boundaries. Assert if searching a key that is
+ // less than first key in the mapfile, that the first key is returned.
+ key = new Text("00");
+ closest = (Text) reader.getClosest(key, value);
+ assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
+
+ // Assert that null is returned if key is > last entry in mapfile.
+ key = new Text("92");
+ closest = (Text) reader.getClosest(key, value);
+ assertNull("Not null key in testGetClosestWithNewCode", closest);
+
+ // If we were looking for the key before, we should get the last key
+ closest = (Text) reader.getClosest(key, value, true);
+ assertEquals(new Text("91"), closest);
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
+ }
}
/**
@@ -187,16 +194,21 @@ public class TestMapFile {
public void testMidKeyOnCurrentApi() throws Exception {
// Write a mapfile of simple data: keys are
final String TEST_PREFIX = "testMidKeyOnCurrentApi.mapfile";
- MapFile.Writer writer = createWriter(TEST_PREFIX, IntWritable.class,
- IntWritable.class);
- // 0,1,....9
- int SIZE = 10;
- for (int i = 0; i < SIZE; i++)
- writer.append(new IntWritable(i), new IntWritable(i));
- writer.close();
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
+ try {
+ writer = createWriter(TEST_PREFIX, IntWritable.class, IntWritable.class);
+ // 0,1,....9
+ int SIZE = 10;
+ for (int i = 0; i < SIZE; i++)
+ writer.append(new IntWritable(i), new IntWritable(i));
+ writer.close();
- MapFile.Reader reader = createReader(TEST_PREFIX, IntWritable.class);
- assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey());
+ reader = createReader(TEST_PREFIX, IntWritable.class);
+ assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey());
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
+ }
}
/**
@@ -206,16 +218,18 @@ public class TestMapFile {
public void testRename() {
final String NEW_FILE_NAME = "test-new.mapfile";
final String OLD_FILE_NAME = "test-old.mapfile";
+ MapFile.Writer writer = null;
try {
FileSystem fs = FileSystem.getLocal(conf);
- MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
- IntWritable.class);
+ writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
writer.close();
MapFile.rename(fs, new Path(TEST_DIR, OLD_FILE_NAME).toString(),
new Path(TEST_DIR, NEW_FILE_NAME).toString());
MapFile.delete(fs, new Path(TEST_DIR, NEW_FILE_NAME).toString());
} catch (IOException ex) {
fail("testRename error " + ex);
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -228,12 +242,12 @@ public class TestMapFile {
final String ERROR_MESSAGE = "Can't rename file";
final String NEW_FILE_NAME = "test-new.mapfile";
final String OLD_FILE_NAME = "test-old.mapfile";
+ MapFile.Writer writer = null;
try {
FileSystem fs = FileSystem.getLocal(conf);
FileSystem spyFs = spy(fs);
- MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
- IntWritable.class);
+ writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
writer.close();
Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
@@ -246,6 +260,8 @@ public class TestMapFile {
} catch (IOException ex) {
assertEquals("testRenameWithException invalid IOExceptionMessage !!!",
ex.getMessage(), ERROR_MESSAGE);
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -254,12 +270,12 @@ public class TestMapFile {
final String ERROR_MESSAGE = "Could not rename";
final String NEW_FILE_NAME = "test-new.mapfile";
final String OLD_FILE_NAME = "test-old.mapfile";
+ MapFile.Writer writer = null;
try {
FileSystem fs = FileSystem.getLocal(conf);
FileSystem spyFs = spy(fs);
- MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
- IntWritable.class);
+ writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
writer.close();
Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
@@ -271,6 +287,8 @@ public class TestMapFile {
} catch (IOException ex) {
assertTrue("testRenameWithFalse invalid IOExceptionMessage error !!!", ex
.getMessage().startsWith(ERROR_MESSAGE));
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -297,11 +315,7 @@ public class TestMapFile {
assertTrue("testWriteWithFailDirCreation ex error !!!", ex.getMessage()
.startsWith(ERROR_MESSAGE));
} finally {
- if (writer != null)
- try {
- writer.close();
- } catch (IOException e) {
- }
+ IOUtils.cleanup(null, writer);
}
}
@@ -312,20 +326,24 @@ public class TestMapFile {
public void testOnFinalKey() {
final String TEST_METHOD_KEY = "testOnFinalKey.mapfile";
int SIZE = 10;
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
try {
- MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
- IntWritable.class);
+ writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
+ IntWritable.class);
for (int i = 0; i < SIZE; i++)
writer.append(new IntWritable(i), new IntWritable(i));
writer.close();
- MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class);
+ reader = createReader(TEST_METHOD_KEY, IntWritable.class);
IntWritable expectedKey = new IntWritable(0);
reader.finalKey(expectedKey);
assertEquals("testOnFinalKey not same !!!", expectedKey, new IntWritable(
9));
} catch (IOException ex) {
fail("testOnFinalKey error !!!");
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -338,7 +356,8 @@ public class TestMapFile {
Class<? extends WritableComparable<?>> keyClass = IntWritable.class;
Class<?> valueClass = Text.class;
try {
- createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class);
+ createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class)
+ .close();
assertNotNull("writer key class null error !!!",
MapFile.Writer.keyClass(keyClass));
assertNotNull("writer value class null error !!!",
@@ -354,19 +373,22 @@ public class TestMapFile {
@Test
public void testReaderGetClosest() throws Exception {
final String TEST_METHOD_KEY = "testReaderWithWrongKeyClass.mapfile";
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
try {
- MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
- Text.class);
+ writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
for (int i = 0; i < 10; i++)
writer.append(new IntWritable(i), new Text("value" + i));
writer.close();
- MapFile.Reader reader = createReader(TEST_METHOD_KEY, Text.class);
+ reader = createReader(TEST_METHOD_KEY, Text.class);
reader.getClosest(new Text("2"), new Text(""));
fail("no excepted exception in testReaderWithWrongKeyClass !!!");
} catch (IOException ex) {
/* Should be thrown to pass the test */
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -376,13 +398,15 @@ public class TestMapFile {
@Test
public void testReaderWithWrongValueClass() {
final String TEST_METHOD_KEY = "testReaderWithWrongValueClass.mapfile";
+ MapFile.Writer writer = null;
try {
- MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
- Text.class);
+ writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
writer.append(new IntWritable(0), new IntWritable(0));
fail("no excepted exception in testReaderWithWrongKeyClass !!!");
} catch (IOException ex) {
/* Should be thrown to pass the test */
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -394,15 +418,16 @@ public class TestMapFile {
final String TEST_METHOD_KEY = "testReaderKeyIteration.mapfile";
int SIZE = 10;
int ITERATIONS = 5;
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
try {
- MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
- Text.class);
+ writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
int start = 0;
for (int i = 0; i < SIZE; i++)
writer.append(new IntWritable(i), new Text("Value:" + i));
writer.close();
- MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class);
+ reader = createReader(TEST_METHOD_KEY, IntWritable.class);
// test iteration
Writable startValue = new Text("Value:" + start);
int i = 0;
@@ -421,6 +446,8 @@ public class TestMapFile {
reader.seek(new IntWritable(SIZE * 2)));
} catch (IOException ex) {
fail("reader seek error !!!");
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -431,11 +458,11 @@ public class TestMapFile {
public void testFix() {
final String INDEX_LESS_MAP_FILE = "testFix.mapfile";
int PAIR_SIZE = 20;
+ MapFile.Writer writer = null;
try {
FileSystem fs = FileSystem.getLocal(conf);
Path dir = new Path(TEST_DIR, INDEX_LESS_MAP_FILE);
- MapFile.Writer writer = createWriter(INDEX_LESS_MAP_FILE,
- IntWritable.class, Text.class);
+ writer = createWriter(INDEX_LESS_MAP_FILE, IntWritable.class, Text.class);
for (int i = 0; i < PAIR_SIZE; i++)
writer.append(new IntWritable(0), new Text("value"));
writer.close();
@@ -450,6 +477,8 @@ public class TestMapFile {
MapFile.fix(fs, dir, IntWritable.class, Text.class, true, conf) == PAIR_SIZE);
} catch (Exception ex) {
fail("testFix error !!!");
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
/**
@@ -459,38 +488,46 @@ public class TestMapFile {
@SuppressWarnings("deprecation")
public void testDeprecatedConstructors() {
String path = new Path(TEST_DIR, "writes.mapfile").toString();
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
try {
FileSystem fs = FileSystem.getLocal(conf);
- MapFile.Writer writer = new MapFile.Writer(conf, fs, path,
+ writer = new MapFile.Writer(conf, fs, path,
IntWritable.class, Text.class, CompressionType.RECORD);
assertNotNull(writer);
+ writer.close();
writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
Text.class, CompressionType.RECORD, defaultProgressable);
assertNotNull(writer);
+ writer.close();
writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
Text.class, CompressionType.RECORD, defaultCodec, defaultProgressable);
assertNotNull(writer);
+ writer.close();
writer = new MapFile.Writer(conf, fs, path,
WritableComparator.get(Text.class), Text.class);
assertNotNull(writer);
+ writer.close();
writer = new MapFile.Writer(conf, fs, path,
WritableComparator.get(Text.class), Text.class,
SequenceFile.CompressionType.RECORD);
assertNotNull(writer);
+ writer.close();
writer = new MapFile.Writer(conf, fs, path,
WritableComparator.get(Text.class), Text.class,
CompressionType.RECORD, defaultProgressable);
assertNotNull(writer);
writer.close();
- MapFile.Reader reader = new MapFile.Reader(fs, path,
+ reader = new MapFile.Reader(fs, path,
WritableComparator.get(IntWritable.class), conf);
assertNotNull(reader);
assertNotNull("reader key is null !!!", reader.getKeyClass());
assertNotNull("reader value in null", reader.getValueClass());
-
} catch (IOException e) {
fail(e.getMessage());
+ } finally {
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -509,11 +546,7 @@ public class TestMapFile {
} catch (Exception e) {
fail("fail in testKeyLessWriterCreation. Other ex !!!");
} finally {
- if (writer != null)
- try {
- writer.close();
- } catch (IOException e) {
- }
+ IOUtils.cleanup(null, writer);
}
}
/**
@@ -542,11 +575,7 @@ public class TestMapFile {
} catch (Exception e) {
fail("fail in testPathExplosionWriterCreation. Other ex !!!");
} finally {
- if (writer != null)
- try {
- writer.close();
- } catch (IOException e) {
- }
+ IOUtils.cleanup(null, writer);
}
}
@@ -555,9 +584,9 @@ public class TestMapFile {
*/
@Test
public void testDescOrderWithThrowExceptionWriterAppend() {
+ MapFile.Writer writer = null;
try {
- MapFile.Writer writer = createWriter(".mapfile", IntWritable.class,
- Text.class);
+ writer = createWriter(".mapfile", IntWritable.class, Text.class);
writer.append(new IntWritable(2), new Text("value: " + 1));
writer.append(new IntWritable(2), new Text("value: " + 2));
writer.append(new IntWritable(2), new Text("value: " + 4));
@@ -566,6 +595,8 @@ public class TestMapFile {
} catch (IOException ex) {
} catch (Exception e) {
fail("testDescOrderWithThrowExceptionWriterAppend other ex throw !!!");
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -575,15 +606,17 @@ public class TestMapFile {
String inFile = "mainMethodMapFile.mapfile";
String outFile = "mainMethodMapFile.mapfile";
String[] args = { path, outFile };
+ MapFile.Writer writer = null;
try {
- MapFile.Writer writer = createWriter(inFile, IntWritable.class,
- Text.class);
+ writer = createWriter(inFile, IntWritable.class, Text.class);
writer.append(new IntWritable(1), new Text("test_text1"));
writer.append(new IntWritable(2), new Text("test_text2"));
writer.close();
MapFile.main(args);
} catch (Exception ex) {
fail("testMainMethodMapFile error !!!");
+ } finally {
+ IOUtils.cleanup(null, writer);
}
}
@@ -601,56 +634,58 @@ public class TestMapFile {
Path qualifiedDirName = fs.makeQualified(dirName);
// Make an index entry for every third insertion.
MapFile.Writer.setIndexInterval(conf, 3);
- MapFile.Writer writer = new MapFile.Writer(conf, fs,
- qualifiedDirName.toString(), Text.class, Text.class);
- // Assert that the index interval is 1
- assertEquals(3, writer.getIndexInterval());
- // Add entries up to 100 in intervals of ten.
- final int FIRST_KEY = 10;
- for (int i = FIRST_KEY; i < 100; i += 10) {
- String iStr = Integer.toString(i);
- Text t = new Text("00".substring(iStr.length()) + iStr);
- writer.append(t, t);
- }
- writer.close();
- // Now do getClosest on created mapfile.
- MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
try {
- Text key = new Text("55");
- Text value = new Text();
- Text closest = (Text) reader.getClosest(key, value);
- // Assert that closest after 55 is 60
- assertEquals(new Text("60"), closest);
- // Get closest that falls before the passed key: 50
- closest = (Text) reader.getClosest(key, value, true);
- assertEquals(new Text("50"), closest);
- // Test get closest when we pass explicit key
- final Text TWENTY = new Text("20");
- closest = (Text) reader.getClosest(TWENTY, value);
- assertEquals(TWENTY, closest);
- closest = (Text) reader.getClosest(TWENTY, value, true);
- assertEquals(TWENTY, closest);
- // Test what happens at boundaries. Assert if searching a key that is
- // less than first key in the mapfile, that the first key is returned.
- key = new Text("00");
- closest = (Text) reader.getClosest(key, value);
- assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
-
- // If we're looking for the first key before, and we pass in a key before
- // the first key in the file, we should get null
- closest = (Text) reader.getClosest(key, value, true);
- assertNull(closest);
-
- // Assert that null is returned if key is > last entry in mapfile.
- key = new Text("99");
- closest = (Text) reader.getClosest(key, value);
- assertNull(closest);
-
- // If we were looking for the key before, we should get the last key
- closest = (Text) reader.getClosest(key, value, true);
- assertEquals(new Text("90"), closest);
+ writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
+ Text.class, Text.class);
+ // Assert that the index interval is 1
+ assertEquals(3, writer.getIndexInterval());
+ // Add entries up to 100 in intervals of ten.
+ final int FIRST_KEY = 10;
+ for (int i = FIRST_KEY; i < 100; i += 10) {
+ String iStr = Integer.toString(i);
+ Text t = new Text("00".substring(iStr.length()) + iStr);
+ writer.append(t, t);
+ }
+ writer.close();
+ // Now do getClosest on created mapfile.
+ reader = new MapFile.Reader(qualifiedDirName, conf);
+ Text key = new Text("55");
+ Text value = new Text();
+ Text closest = (Text) reader.getClosest(key, value);
+ // Assert that closest after 55 is 60
+ assertEquals(new Text("60"), closest);
+ // Get closest that falls before the passed key: 50
+ closest = (Text) reader.getClosest(key, value, true);
+ assertEquals(new Text("50"), closest);
+ // Test get closest when we pass explicit key
+ final Text TWENTY = new Text("20");
+ closest = (Text) reader.getClosest(TWENTY, value);
+ assertEquals(TWENTY, closest);
+ closest = (Text) reader.getClosest(TWENTY, value, true);
+ assertEquals(TWENTY, closest);
+ // Test what happens at boundaries. Assert if searching a key that is
+ // less than first key in the mapfile, that the first key is returned.
+ key = new Text("00");
+ closest = (Text) reader.getClosest(key, value);
+ assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
+
+ // If we're looking for the first key before, and we pass in a key before
+ // the first key in the file, we should get null
+ closest = (Text) reader.getClosest(key, value, true);
+ assertNull(closest);
+
+ // Assert that null is returned if key is > last entry in mapfile.
+ key = new Text("99");
+ closest = (Text) reader.getClosest(key, value);
+ assertNull(closest);
+
+ // If we were looking for the key before, we should get the last key
+ closest = (Text) reader.getClosest(key, value, true);
+ assertEquals(new Text("90"), closest);
} finally {
- reader.close();
+ IOUtils.cleanup(null, writer, reader);
}
}
@@ -662,16 +697,18 @@ public class TestMapFile {
FileSystem fs = FileSystem.getLocal(conf);
Path qualifiedDirName = fs.makeQualified(dirName);
- MapFile.Writer writer = new MapFile.Writer(conf, fs,
- qualifiedDirName.toString(), IntWritable.class, IntWritable.class);
- writer.append(new IntWritable(1), new IntWritable(1));
- writer.close();
- // Now do getClosest on created mapfile.
- MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
+ MapFile.Writer writer = null;
+ MapFile.Reader reader = null;
try {
+ writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
+ IntWritable.class, IntWritable.class);
+ writer.append(new IntWritable(1), new IntWritable(1));
+ writer.close();
+ // Now do getClosest on created mapfile.
+ reader = new MapFile.Reader(qualifiedDirName, conf);
assertEquals(new IntWritable(1), reader.midKey());
} finally {
- reader.close();
+ IOUtils.cleanup(null, writer, reader);
}
}