You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/03/23 17:37:58 UTC

[1/3] flink git commit: [FLINK-2445] Add tests for HadoopOutputFormats

Repository: flink
Updated Branches:
  refs/heads/release-1.0 1909bdb41 -> 1ca4f36d0


[FLINK-2445] Add tests for HadoopOutputFormats


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/edf343ab
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/edf343ab
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/edf343ab

Branch: refs/heads/release-1.0
Commit: edf343abb6d026d1e2ddb9ba8aaadeca1139bea5
Parents: 1909bdb
Author: Ajay Bhat <a....@gmail.com>
Authored: Wed Jan 6 16:12:03 2016 +0530
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Mar 23 17:36:57 2016 +0100

----------------------------------------------------------------------
 .../mapreduce/HadoopOutputFormatTest.java       | 130 +++++++++++++++++++
 1 file changed, 130 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/edf343ab/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
new file mode 100644
index 0000000..215ca95
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.hadoop.mapreduce;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.*;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class HadoopOutputFormatTest {
+
+    private static final String PATH = "an/ignored/file/";
+    private Map<String, Long> map;
+
+    @Test
+    public void testWriteRecord() {
+        OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
+        String key = "Test";
+        Long value = 1L;
+        map = new HashMap<>();
+        map.put(key, 0L);
+        try {
+            Job job = Job.getInstance();
+            Tuple2<String, Long> tuple = new Tuple2<>();
+            tuple.setFields(key, value);
+            HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
+
+            hadoopOutputFormat.recordWriter = new DummyRecordWriter();
+            hadoopOutputFormat.writeRecord(tuple);
+
+            Long expected = map.get(key);
+            assertEquals(expected, value);
+        } catch (IOException e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void testOpen() {
+        OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
+        try {
+            Job job = Job.getInstance();
+            HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
+
+            hadoopOutputFormat.recordWriter = new DummyRecordWriter();
+            hadoopOutputFormat.open(1, 4);
+        } catch (IOException e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void testClose() {
+        OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
+        try {
+            Job job = Job.getInstance();
+            HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
+
+            hadoopOutputFormat.recordWriter = new DummyRecordWriter();
+
+            final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+            Mockito.when(outputCommitter.needsTaskCommit(Mockito.any(TaskAttemptContext.class))).thenReturn(true);
+            Mockito.doNothing().when(outputCommitter).commitTask(Mockito.any(TaskAttemptContext.class));
+            hadoopOutputFormat.outputCommitter = outputCommitter;
+            hadoopOutputFormat.configuration = new Configuration();
+            hadoopOutputFormat.configuration.set("mapred.output.dir", PATH);
+
+            hadoopOutputFormat.close();
+        } catch (IOException e) {
+            fail();
+        }
+    }
+
+
+    class DummyRecordWriter extends RecordWriter<String, Long> {
+        @Override
+        public void write(String key, Long value) throws IOException, InterruptedException {
+            map.put(key, value);
+        }
+
+        @Override
+        public void close(TaskAttemptContext context) throws IOException, InterruptedException {
+
+        }
+    }
+
+    class DummyOutputFormat extends OutputFormat<String, Long> {
+        @Override
+        public RecordWriter<String, Long> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+            return null;
+        }
+
+        @Override
+        public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
+
+        }
+
+        @Override
+        public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
+            final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+            Mockito.doNothing().when(outputCommitter).setupJob(Mockito.any(JobContext.class));
+
+            return outputCommitter;
+        }
+    }
+}


[3/3] flink git commit: [FLINK-2444] Add tests for HadoopInputFormats

Posted by fh...@apache.org.
[FLINK-2444] Add tests for HadoopInputFormats


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1ca4f36d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1ca4f36d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1ca4f36d

Branch: refs/heads/release-1.0
Commit: 1ca4f36d047cdefe04e12554a7df493aba681be3
Parents: da989c4
Author: Martin Liesenberg <ma...@gmail.com>
Authored: Fri Feb 12 00:18:06 2016 +0100
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Mar 23 17:37:31 2016 +0100

----------------------------------------------------------------------
 .../hadoop/mapred/HadoopInputFormatBase.java    |   4 +-
 .../hadoop/mapreduce/HadoopInputFormat.java     |  10 +-
 .../hadoop/mapreduce/HadoopInputFormatBase.java |  12 +-
 .../hadoop/mapred/HadoopInputFormatTest.java    | 275 +++++++++++++++++--
 .../hadoop/mapreduce/HadoopInputFormatTest.java | 202 ++++++++++++--
 5 files changed, 443 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
index d80c80a..ef9999f 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
@@ -148,7 +148,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 			throws IOException {
 		org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
 		HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
-		for(int i=0;i<splitArray.length;i++){
+		for (int i=0; i<splitArray.length; i++) {
 			hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
 		}
 		return hiSplit;
@@ -177,7 +177,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 	
 	@Override
 	public boolean reachedEnd() throws IOException {
-		if(!fetched) {
+		if (!fetched) {
 			fetchNext();
 		}
 		return !hasNext;

http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
index dadb954..abbcd25 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
@@ -18,15 +18,15 @@
 
 package org.apache.flink.api.java.hadoop.mapreduce;
 
-import java.io.IOException;
-
 import org.apache.flink.annotation.Public;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.hadoop.mapreduce.Job;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.hadoop.mapreduce.Job;
+
+import java.io.IOException;
 
 /**
  * InputFormat implementation allowing to use Hadoop (mapreduce) InputFormats with Flink.
@@ -49,10 +49,10 @@ public class HadoopInputFormat<K, V> extends HadoopInputFormatBase<K, V, Tuple2<
 
 	@Override
 	public Tuple2<K, V> nextRecord(Tuple2<K, V> record) throws IOException {
-		if(!this.fetched) {
+		if (!this.fetched) {
 			fetchNext();
 		}
-		if(!this.hasNext) {
+		if (!this.hasNext) {
 			return null;
 		}
 		try {

http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
index 4829ce7..73b11eb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
@@ -111,7 +111,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 	@Override
 	public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
 		// only gather base statistics for FileInputFormats
-		if(!(mapreduceInputFormat instanceof FileInputFormat)) {
+		if (!(mapreduceInputFormat instanceof FileInputFormat)) {
 			return null;
 		}
 
@@ -158,7 +158,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 
 		jobContext.getCredentials().addAll(this.credentials);
 		Credentials currentUserCreds = getCredentialsFromUGI(UserGroupInformation.getCurrentUser());
-		if(currentUserCreds != null) {
+		if (currentUserCreds != null) {
 			jobContext.getCredentials().addAll(currentUserCreds);
 		}
 
@@ -170,7 +170,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 		}
 		HadoopInputSplit[] hadoopInputSplits = new HadoopInputSplit[splits.size()];
 
-		for(int i = 0; i < hadoopInputSplits.length; i++){
+		for (int i = 0; i < hadoopInputSplits.length; i++) {
 			hadoopInputSplits[i] = new HadoopInputSplit(i, splits.get(i), jobContext);
 		}
 		return hadoopInputSplits;
@@ -208,7 +208,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 
 	@Override
 	public boolean reachedEnd() throws IOException {
-		if(!this.fetched) {
+		if (!this.fetched) {
 			fetchNext();
 		}
 		return !this.hasNext;
@@ -243,7 +243,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 		long latestModTime = 0L;
 
 		// get the file info and check whether the cached statistics are still valid.
-		for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
+		for (org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
 
 			final Path filePath = new Path(hadoopPath.toUri());
 			final FileSystem fs = FileSystem.get(filePath.toUri());
@@ -308,7 +308,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
 		org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
 		configuration.readFields(in);
 
-		if(this.configuration == null) {
+		if (this.configuration == null) {
 			this.configuration = configuration;
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
index 63f8df4..3b8d227 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
@@ -20,21 +20,168 @@ package org.apache.flink.api.java.hadoop.mapred;
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.hadoop.mapred.HadoopInputFormat;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobConfigurable;
+import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.FileInputFormat;
 import org.junit.Test;
 
 import java.io.IOException;
 
-import static org.junit.Assert.fail;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 
 public class HadoopInputFormatTest {
 
+	@Test
+	public void testConfigureWithConfigurableInstance() {
+		ConfigurableDummyInputFormat inputFormat = mock(ConfigurableDummyInputFormat.class);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+		verify(inputFormat, times(1)).setConf(any(JobConf.class));
+
+		hadoopInputFormat.configure(new org.apache.flink.configuration.Configuration());
+		verify(inputFormat, times(2)).setConf(any(JobConf.class));
+	}
+
+	@Test
+	public void testConfigureWithJobConfigurableInstance() {
+		JobConfigurableDummyInputFormat inputFormat = mock(JobConfigurableDummyInputFormat.class);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+		verify(inputFormat, times(1)).configure(any(JobConf.class));
+
+		hadoopInputFormat.configure(new org.apache.flink.configuration.Configuration());
+		verify(inputFormat, times(2)).configure(any(JobConf.class));
+	}
+
+	@Test
+	public void testOpenClose() throws Exception {
+		DummyRecordReader recordReader = mock(DummyRecordReader.class);
+		DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+		when(inputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+		hadoopInputFormat.open(getHadoopInputSplit());
+
+		verify(inputFormat, times(1)).getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class));
+		verify(recordReader, times(1)).createKey();
+		verify(recordReader, times(1)).createValue();
+
+		assertThat(hadoopInputFormat.fetched, is(false));
+
+		hadoopInputFormat.close();
+		verify(recordReader, times(1)).close();
+	}
+
+	@Test
+	public void testOpenWithConfigurableReader() throws Exception {
+		ConfigurableDummyRecordReader recordReader = mock(ConfigurableDummyRecordReader.class);
+		DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+		when(inputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+		hadoopInputFormat.open(getHadoopInputSplit());
+
+		verify(inputFormat, times(1)).getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class));
+		verify(recordReader, times(1)).setConf(any(JobConf.class));
+		verify(recordReader, times(1)).createKey();
+		verify(recordReader, times(1)).createValue();
+
+		assertThat(hadoopInputFormat.fetched, is(false));
+
+	}
+
+	@Test
+	public void testCreateInputSplits() throws Exception {
+
+		FileSplit[] result = new FileSplit[1];
+		result[0] = getFileSplit();
+		DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+		when(inputFormat.getSplits(any(JobConf.class), anyInt())).thenReturn(result);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+		hadoopInputFormat.createInputSplits(2);
+
+		verify(inputFormat, times(1)).getSplits(any(JobConf.class), anyInt());
+	}
+
+	@Test
+	public void testReachedEndWithElementsRemaining() throws IOException {
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(new DummyInputFormat(), String.class, Long.class, new JobConf());
+		hadoopInputFormat.fetched = true;
+		hadoopInputFormat.hasNext = true;
+
+		assertThat(hadoopInputFormat.reachedEnd(), is(false));
+	}
+
+	@Test
+	public void testReachedEndWithNoElementsRemaining() throws IOException {
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(new DummyInputFormat(), String.class, Long.class, new JobConf());
+		hadoopInputFormat.fetched = true;
+		hadoopInputFormat.hasNext = false;
+
+		assertThat(hadoopInputFormat.reachedEnd(), is(true));
+	}
+
+	@Test
+	public void testFetchNext() throws IOException {
+		DummyRecordReader recordReader = mock(DummyRecordReader.class);
+		when(recordReader.next(anyString(), anyLong())).thenReturn(true);
+
+		DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+		when(inputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+		hadoopInputFormat.open(getHadoopInputSplit());
+		hadoopInputFormat.fetchNext();
+
+		verify(recordReader, times(1)).next(anyString(), anyLong());
+		assertThat(hadoopInputFormat.hasNext, is(true));
+		assertThat(hadoopInputFormat.fetched, is(true));
+	}
+	
+	@Test
+	public void checkTypeInformation() throws Exception {
+		HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<>(
+				new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, new JobConf());
+
+		TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
+		TypeInformation<Tuple2<Void,Long>> expectedType = new TupleTypeInfo<>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
+
+		assertThat(tupleType.isTupleType(), is(true));
+		assertThat(tupleType, is(equalTo(expectedType)));
+	}
+
+	private HadoopInputSplit getHadoopInputSplit() {
+		return new HadoopInputSplit(1, getFileSplit(), new JobConf());
+	}
+
+	private FileSplit getFileSplit() {
+		return new FileSplit(new Path("path"), 1, 2, new String[]{});
+	}
+
 	public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
 
 		public DummyVoidKeyInputFormat() {}
@@ -44,27 +191,107 @@ public class HadoopInputFormatTest {
 			return null;
 		}
 	}
-	
-	@Test
-	public void checkTypeInformation() {
-		try {
-			HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>(
-					new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, new JobConf());
-
-			TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
-			TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
-			
-			if(tupleType.isTupleType()) {
-				if(!((TupleTypeInfo<?>)tupleType).equals(testTupleType)) {
-					fail("Tuple type information was not set correctly!");
-				}
-			} else {
-				fail("Type information was not set to tuple type information!");
-			}
-
-		}
-		catch (Exception ex) {
-			fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
+
+	public class DummyRecordReader implements RecordReader<String, Long> {
+
+		@Override
+		public float getProgress() throws IOException {
+			return 0;
+		}
+
+		@Override
+		public boolean next(String s, Long aLong) throws IOException {
+			return false;
 		}
+
+		@Override
+		public String createKey() {
+			return null;
+		}
+
+		@Override
+		public Long createValue() {
+			return null;
+		}
+
+		@Override
+		public long getPos() throws IOException {
+			return 0;
+		}
+
+		@Override
+		public void close() throws IOException {
+
+		}
+	}
+
+	public class ConfigurableDummyRecordReader implements RecordReader<String, Long>, Configurable {
+
+		@Override
+		public void setConf(Configuration configuration) {}
+
+		@Override
+		public Configuration getConf() {
+			return null;
+		}
+
+		@Override
+		public boolean next(String s, Long aLong) throws IOException {
+			return false;
+		}
+
+		@Override
+		public String createKey() {
+			return null;
+		}
+
+		@Override
+		public Long createValue() {
+			return null;
+		}
+
+		@Override
+		public long getPos() throws IOException {
+			return 0;
+		}
+
+		@Override
+		public void close() throws IOException {
+
+		}
+
+		@Override
+		public float getProgress() throws IOException {
+			return 0;
+		}
+	}
+
+	public class DummyInputFormat implements InputFormat<String, Long> {
+
+		@Override
+		public InputSplit[] getSplits(JobConf jobConf, int i) throws IOException {
+			return new InputSplit[0];
+		}
+
+		@Override
+		public RecordReader<String, Long> getRecordReader(InputSplit inputSplit, JobConf jobConf, Reporter reporter) throws IOException {
+			return null;
+		}
+	}
+
+	public class ConfigurableDummyInputFormat extends DummyInputFormat implements Configurable {
+		@Override
+		public void setConf(Configuration configuration) {}
+
+		@Override
+		public Configuration getConf() {
+			return null;
+		}
+	}
+
+	public class JobConfigurableDummyInputFormat extends DummyInputFormat implements JobConfigurable {
+
+		@Override
+		public void configure(JobConf jobConf) {}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
index 919a286..d6ec484 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
@@ -20,22 +20,146 @@ package org.apache.flink.api.java.hadoop.mapreduce;
 
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.hadoop.mapreduce.wrapper.HadoopInputSplit;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
+import java.util.List;
 
-import static org.junit.Assert.fail;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class HadoopInputFormatTest {
 
+	@Rule
+	public final ExpectedException exception = ExpectedException.none();
+
+	@Test
+	public void testConfigure() throws Exception {
+
+		ConfigurableDummyInputFormat inputFormat = mock(ConfigurableDummyInputFormat.class);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(inputFormat, Job.getInstance(), null);
+		hadoopInputFormat.configure(new org.apache.flink.configuration.Configuration());
+
+		verify(inputFormat, times(1)).setConf(any(Configuration.class));
+	}
+
+	@Test
+	public void testCreateInputSplits() throws Exception {
+		DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(inputFormat, Job.getInstance(), null);
+		hadoopInputFormat.createInputSplits(2);
+
+		verify(inputFormat, times(1)).getSplits(any(JobContext.class));
+	}
+
+	@Test
+	public void testOpen() throws Exception {
+		DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+		when(inputFormat.createRecordReader(any(InputSplit.class), any(TaskAttemptContext.class))).thenReturn(new DummyRecordReader());
+		HadoopInputSplit inputSplit = mock(HadoopInputSplit.class);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(inputFormat, Job.getInstance(), null);
+		hadoopInputFormat.open(inputSplit);
+
+		verify(inputFormat, times(1)).createRecordReader(any(InputSplit.class), any(TaskAttemptContext.class));
+		assertThat(hadoopInputFormat.fetched, is(false));
+	}
+
+	@Test
+	public void testClose() throws Exception {
+
+		DummyRecordReader recordReader = mock(DummyRecordReader.class);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+		hadoopInputFormat.close();
+
+		verify(recordReader, times(1)).close();
+	}
+
+	@Test
+	public void testFetchNextInitialState() throws Exception {
+		DummyRecordReader recordReader = new DummyRecordReader();
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+		hadoopInputFormat.fetchNext();
+
+		assertThat(hadoopInputFormat.fetched, is(true));
+		assertThat(hadoopInputFormat.hasNext, is(false));
+	}
+
+	@Test
+	public void testFetchNextRecordReaderHasNewValue() throws Exception {
+
+		DummyRecordReader recordReader = mock(DummyRecordReader.class);
+		when(recordReader.nextKeyValue()).thenReturn(true);
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+		hadoopInputFormat.fetchNext();
+
+		assertThat(hadoopInputFormat.fetched, is(true));
+		assertThat(hadoopInputFormat.hasNext, is(true));
+	}
+
+	@Test
+	public void testFetchNextRecordReaderThrowsException() throws Exception {
+
+		DummyRecordReader recordReader = mock(DummyRecordReader.class);
+		when(recordReader.nextKeyValue()).thenThrow(new InterruptedException());
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+
+		exception.expect(IOException.class);
+		hadoopInputFormat.fetchNext();
+
+		assertThat(hadoopInputFormat.hasNext, is(true));
+	}
+
+	@Test
+	public void checkTypeInformation() throws Exception {
+
+		HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<>(
+				new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, Job.getInstance());
+
+		TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
+		TypeInformation<Tuple2<Void,Long>> expectedType = new TupleTypeInfo<>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
+
+		assertThat(tupleType.isTupleType(), is(true));
+		assertThat(tupleType, is(equalTo(expectedType)));
+	}
+
+
+	private HadoopInputFormat<String, Long> setupHadoopInputFormat(InputFormat<String, Long> inputFormat, Job job,
+																   RecordReader<String, Long> recordReader) {
+
+		HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat,
+				String.class, Long.class, job);
+		hadoopInputFormat.recordReader = recordReader;
+
+		return hadoopInputFormat;
+	}
+
 	public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
 
 		public DummyVoidKeyInputFormat() {}
@@ -45,29 +169,61 @@ public class HadoopInputFormatTest {
 			return null;
 		}
 	}
-	
-	@Test
-	public void checkTypeInformation() {
-		try {
-			// Set up the Hadoop Input Format
-			Job job = Job.getInstance();
-			HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>(
-					new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, job);
-
-			TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
-			TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
-			
-			if(tupleType.isTupleType()) {
-				if(!((TupleTypeInfo<?>)tupleType).equals(testTupleType)) {
-					fail("Tuple type information was not set correctly!");
-				}
-			} else {
-				fail("Type information was not set to tuple type information!");
-			}
+
+	public class DummyRecordReader extends RecordReader<String, Long> {
+
+		@Override
+		public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
 
 		}
-		catch (Exception ex) {
-			fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
+
+		@Override
+		public boolean nextKeyValue() throws IOException, InterruptedException {
+			return false;
+		}
+
+		@Override
+		public String getCurrentKey() throws IOException, InterruptedException {
+			return null;
+		}
+
+		@Override
+		public Long getCurrentValue() throws IOException, InterruptedException {
+			return null;
+		}
+
+		@Override
+		public float getProgress() throws IOException, InterruptedException {
+			return 0;
+		}
+
+		@Override
+		public void close() throws IOException {
+
+		}
+	}
+
+	public class DummyInputFormat extends InputFormat<String, Long> {
+
+		@Override
+		public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
+			return null;
+		}
+
+		@Override
+		public RecordReader<String, Long> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+			return new DummyRecordReader();
+		}
+	}
+
+	public class ConfigurableDummyInputFormat extends DummyInputFormat implements Configurable {
+
+		@Override
+		public void setConf(Configuration configuration) {}
+
+		@Override
+		public Configuration getConf() {
+			return null;
 		}
 	}
 }


[2/3] flink git commit: [FLINK-2445] Improve HadoopOutputFormatTests

Posted by fh...@apache.org.
[FLINK-2445] Improve HadoopOutputFormatTests


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/da989c42
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/da989c42
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/da989c42

Branch: refs/heads/release-1.0
Commit: da989c423306212646f456d410562d9ec3779b24
Parents: edf343a
Author: Martin Liesenberg <ma...@gmail.com>
Authored: Thu Feb 11 15:59:01 2016 +0100
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Mar 23 17:37:15 2016 +0100

----------------------------------------------------------------------
 .../hadoop/mapred/HadoopOutputFormatTest.java   | 229 ++++++++++++++++
 .../mapreduce/HadoopOutputFormatTest.java       | 265 ++++++++++++-------
 2 files changed, 395 insertions(+), 99 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/da989c42/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java
new file mode 100644
index 0000000..409feb4
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.hadoop.mapred;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobConfigurable;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.util.Progressable;
+import org.junit.Test;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class HadoopOutputFormatTest {
+
+	@Test
+	public void testOpen() throws Exception {
+
+		OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+		DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+		JobConf jobConf = spy(new JobConf());
+		when(jobConf.getOutputCommitter()).thenReturn(outputCommitter);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+		outputFormat.open(1, 1);
+
+		verify(jobConf, times(2)).getOutputCommitter();
+		verify(outputCommitter, times(1)).setupJob(any(JobContext.class));
+		verify(dummyOutputFormat, times(1)).getRecordWriter(any(FileSystem.class), any(JobConf.class), anyString(), any(Progressable.class));
+	}
+
+	@Test
+	public void testConfigureWithConfigurable() {
+		ConfigurableDummyOutputFormat dummyOutputFormat = mock(ConfigurableDummyOutputFormat.class);
+		JobConf jobConf = mock(JobConf.class);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+		outputFormat.configure(Matchers.<org.apache.flink.configuration.Configuration>any());
+
+		verify(dummyOutputFormat, times(1)).setConf(any(Configuration.class));
+	}
+
+	@Test
+	public void testConfigureWithJobConfigurable() {
+		JobConfigurableDummyOutputFormat dummyOutputFormat = mock(JobConfigurableDummyOutputFormat.class);
+		JobConf jobConf = mock(JobConf.class);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+		outputFormat.configure(Matchers.<org.apache.flink.configuration.Configuration>any());
+
+		verify(dummyOutputFormat, times(1)).configure(any(JobConf.class));
+	}
+
+	@Test
+	public void testCloseWithTaskCommit() throws Exception {
+		OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+		DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+		when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(true);
+		DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
+		JobConf jobConf = mock(JobConf.class);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+		outputFormat.recordWriter = recordWriter;
+		outputFormat.outputCommitter = outputCommitter;
+
+		outputFormat.close();
+
+		verify(recordWriter, times(1)).close(any(Reporter.class));
+		verify(outputCommitter, times(1)).commitTask(any(TaskAttemptContext.class));
+	}
+
+	@Test
+	public void testCloseWithoutTaskCommit() throws Exception {
+		OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+		DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+		when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(false);
+		DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
+		JobConf jobConf = mock(JobConf.class);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+		outputFormat.recordWriter = recordWriter;
+		outputFormat.outputCommitter = outputCommitter;
+
+		outputFormat.close();
+
+		verify(recordWriter, times(1)).close(any(Reporter.class));
+		verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
+	}
+
+	@Test
+	public void testWriteRecord() throws Exception {
+		OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+		DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
+		JobConf jobConf = mock(JobConf.class);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+		outputFormat.recordWriter = recordWriter;
+
+		outputFormat.writeRecord(new Tuple2<>("key", 1L));
+
+		verify(recordWriter, times(1)).write(anyString(), anyLong());
+	}
+
+	@Test
+	public void testFinalizeGlobal() throws Exception {
+		OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+		DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+		JobConf jobConf = spy(new JobConf());
+		when(jobConf.getOutputCommitter()).thenReturn(outputCommitter);
+
+		HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+		outputFormat.finalizeGlobal(1);
+
+		verify(outputCommitter, times(1)).commitJob(any(JobContext.class));
+	}
+
+	public class DummyOutputFormat implements OutputFormat<String, Long> {
+
+		@Override
+		public RecordWriter<String, Long> getRecordWriter(FileSystem fileSystem, JobConf jobConf, String s, Progressable progressable) throws IOException {
+			return null;
+		}
+
+		@Override
+		public void checkOutputSpecs(FileSystem fileSystem, JobConf jobConf) throws IOException {
+
+		}
+	}
+
+	public class ConfigurableDummyOutputFormat extends DummyOutputFormat implements Configurable {
+
+		@Override
+		public void setConf(Configuration configuration) {
+
+		}
+
+		@Override
+		public Configuration getConf() {
+			return null;
+		}
+	}
+
+	public class JobConfigurableDummyOutputFormat extends DummyOutputFormat implements JobConfigurable {
+
+		@Override
+		public void configure(JobConf jobConf) {
+
+		}
+	}
+
+	public class DummyOutputCommitter extends OutputCommitter {
+
+		@Override
+		public void setupJob(JobContext jobContext) throws IOException {
+
+		}
+
+		@Override
+		public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+		}
+
+		@Override
+		public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
+			return false;
+		}
+
+		@Override
+		public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+		}
+
+		@Override
+		public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+		}
+	}
+
+	public class DummyRecordWriter implements RecordWriter<String, Long> {
+
+		@Override
+		public void write(String s, Long aLong) throws IOException {
+
+		}
+
+		@Override
+		public void close(Reporter reporter) throws IOException {
+
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/da989c42/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
index 215ca95..7b126fc 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
@@ -19,112 +19,179 @@
 package org.apache.flink.api.java.hadoop.mapreduce;
 
 import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.junit.Test;
 import org.mockito.Mockito;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 
 public class HadoopOutputFormatTest {
 
-    private static final String PATH = "an/ignored/file/";
-    private Map<String, Long> map;
-
-    @Test
-    public void testWriteRecord() {
-        OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
-        String key = "Test";
-        Long value = 1L;
-        map = new HashMap<>();
-        map.put(key, 0L);
-        try {
-            Job job = Job.getInstance();
-            Tuple2<String, Long> tuple = new Tuple2<>();
-            tuple.setFields(key, value);
-            HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
-
-            hadoopOutputFormat.recordWriter = new DummyRecordWriter();
-            hadoopOutputFormat.writeRecord(tuple);
-
-            Long expected = map.get(key);
-            assertEquals(expected, value);
-        } catch (IOException e) {
-            fail();
-        }
-    }
-
-    @Test
-    public void testOpen() {
-        OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
-        try {
-            Job job = Job.getInstance();
-            HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
-
-            hadoopOutputFormat.recordWriter = new DummyRecordWriter();
-            hadoopOutputFormat.open(1, 4);
-        } catch (IOException e) {
-            fail();
-        }
-    }
-
-    @Test
-    public void testClose() {
-        OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
-        try {
-            Job job = Job.getInstance();
-            HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
-
-            hadoopOutputFormat.recordWriter = new DummyRecordWriter();
-
-            final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
-            Mockito.when(outputCommitter.needsTaskCommit(Mockito.any(TaskAttemptContext.class))).thenReturn(true);
-            Mockito.doNothing().when(outputCommitter).commitTask(Mockito.any(TaskAttemptContext.class));
-            hadoopOutputFormat.outputCommitter = outputCommitter;
-            hadoopOutputFormat.configuration = new Configuration();
-            hadoopOutputFormat.configuration.set("mapred.output.dir", PATH);
-
-            hadoopOutputFormat.close();
-        } catch (IOException e) {
-            fail();
-        }
-    }
-
-
-    class DummyRecordWriter extends RecordWriter<String, Long> {
-        @Override
-        public void write(String key, Long value) throws IOException, InterruptedException {
-            map.put(key, value);
-        }
-
-        @Override
-        public void close(TaskAttemptContext context) throws IOException, InterruptedException {
-
-        }
-    }
-
-    class DummyOutputFormat extends OutputFormat<String, Long> {
-        @Override
-        public RecordWriter<String, Long> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
-            return null;
-        }
-
-        @Override
-        public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
-
-        }
-
-        @Override
-        public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
-            final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
-            Mockito.doNothing().when(outputCommitter).setupJob(Mockito.any(JobContext.class));
-
-            return outputCommitter;
-        }
-    }
+	private static final String MAPRED_OUTPUT_PATH = "an/ignored/file/";
+	private static final String MAPRED_OUTPUT_DIR_KEY = "mapred.output.dir";
+
+	@Test
+	public void testWriteRecord() throws Exception {
+
+		RecordWriter<String, Long> recordWriter = mock(DummyRecordWriter.class);
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+			Job.getInstance(), recordWriter, null, new Configuration());
+
+		hadoopOutputFormat.writeRecord(new Tuple2<String, Long>());
+
+		verify(recordWriter, times(1)).write(anyString(), anyLong());
+	}
+
+	@Test
+	public void testOpen() throws Exception {
+
+
+		OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+		OutputCommitter outputCommitter = setupOutputCommitter(true);
+		when(dummyOutputFormat.getOutputCommitter(any(TaskAttemptContext.class))).thenReturn(outputCommitter);
+
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(dummyOutputFormat,
+			Job.getInstance(), new DummyRecordWriter(), setupOutputCommitter(true), new Configuration());
+
+		hadoopOutputFormat.open(1, 4);
+
+		verify(hadoopOutputFormat.outputCommitter, times(1)).setupJob(any(JobContext.class));
+		verify(hadoopOutputFormat.mapreduceOutputFormat, times(1)).getRecordWriter(any(TaskAttemptContext.class));
+	}
+
+	@Test
+	public void testCloseWithNeedsTaskCommitTrue() throws Exception {
+
+		RecordWriter<String, Long> recordWriter = Mockito.mock(DummyRecordWriter.class);
+		OutputCommitter outputCommitter = setupOutputCommitter(true);
+
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+			Job.getInstance(), recordWriter, outputCommitter, new Configuration());
+
+		hadoopOutputFormat.close();
+
+		verify(outputCommitter, times(1)).commitTask(any(TaskAttemptContext.class));
+		verify(recordWriter, times(1)).close(any(TaskAttemptContext.class));
+	}
+
+	@Test
+	public void testCloseWithNeedsTaskCommitFalse() throws Exception {
+
+		RecordWriter<String, Long> recordWriter = Mockito.mock(DummyRecordWriter.class);
+		OutputCommitter outputCommitter = setupOutputCommitter(false);
+
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+			Job.getInstance(), recordWriter, outputCommitter, new Configuration());
+
+		hadoopOutputFormat.close();
+
+		verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
+		verify(recordWriter, times(1)).close(any(TaskAttemptContext.class));
+	}
+
+	@Test
+	public void testConfigure() throws Exception {
+
+		ConfigurableDummyOutputFormat outputFormat = mock(ConfigurableDummyOutputFormat.class);
+
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(outputFormat, Job.getInstance(),
+			null, null, new Configuration());
+
+		hadoopOutputFormat.configure(new org.apache.flink.configuration.Configuration());
+
+		verify(outputFormat, times(1)).setConf(any(Configuration.class));
+	}
+
+	@Test
+	public void testFinalizedGlobal() throws Exception {
+
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+			Job.getInstance(), null, null, new Configuration());
+
+		hadoopOutputFormat.finalizeGlobal(1);
+
+		verify(hadoopOutputFormat.outputCommitter, times(1)).commitJob(any(JobContext.class));
+	}
+
+	private OutputCommitter setupOutputCommitter(boolean needsTaskCommit) throws IOException {
+		OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+		when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(needsTaskCommit);
+		doNothing().when(outputCommitter).commitTask(any(TaskAttemptContext.class));
+
+		return outputCommitter;
+	}
+
+	private HadoopOutputFormat<String, Long> setupHadoopOutputFormat(OutputFormat<String, Long> outputFormat,
+																	 Job job,
+																	 RecordWriter<String, Long> recordWriter,
+																	 OutputCommitter outputCommitter,
+																	 Configuration configuration) {
+
+		HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(outputFormat, job);
+		hadoopOutputFormat.recordWriter = recordWriter;
+		hadoopOutputFormat.outputCommitter = outputCommitter;
+		hadoopOutputFormat.configuration = configuration;
+		hadoopOutputFormat.configuration.set(MAPRED_OUTPUT_DIR_KEY, MAPRED_OUTPUT_PATH);
+
+		return hadoopOutputFormat;
+	}
+
+	class DummyRecordWriter extends RecordWriter<String, Long> {
+		@Override
+		public void write(String key, Long value) throws IOException, InterruptedException {
+		}
+
+		@Override
+		public void close(TaskAttemptContext context) throws IOException, InterruptedException {
+
+		}
+	}
+
+	class DummyOutputFormat extends OutputFormat<String, Long> {
+		@Override
+		public RecordWriter<String, Long> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+			return null;
+		}
+
+		@Override
+		public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
+
+		}
+
+		@Override
+		public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
+			final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+			doNothing().when(outputCommitter).setupJob(any(JobContext.class));
+
+			return outputCommitter;
+		}
+	}
+
+	class ConfigurableDummyOutputFormat extends DummyOutputFormat implements Configurable {
+
+		@Override
+		public void setConf(Configuration configuration) {}
+
+		@Override
+		public Configuration getConf() {
+			return null;
+		}
+	}
 }