You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/03/23 17:37:58 UTC
[1/3] flink git commit: [FLINK-2445] Add tests for HadoopOutputFormats
Repository: flink
Updated Branches:
refs/heads/release-1.0 1909bdb41 -> 1ca4f36d0
[FLINK-2445] Add tests for HadoopOutputFormats
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/edf343ab
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/edf343ab
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/edf343ab
Branch: refs/heads/release-1.0
Commit: edf343abb6d026d1e2ddb9ba8aaadeca1139bea5
Parents: 1909bdb
Author: Ajay Bhat <a....@gmail.com>
Authored: Wed Jan 6 16:12:03 2016 +0530
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Mar 23 17:36:57 2016 +0100
----------------------------------------------------------------------
.../mapreduce/HadoopOutputFormatTest.java | 130 +++++++++++++++++++
1 file changed, 130 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/edf343ab/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
new file mode 100644
index 0000000..215ca95
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.hadoop.mapreduce;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.*;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class HadoopOutputFormatTest {
+
+ private static final String PATH = "an/ignored/file/";
+ private Map<String, Long> map;
+
+ @Test
+ public void testWriteRecord() {
+ OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
+ String key = "Test";
+ Long value = 1L;
+ map = new HashMap<>();
+ map.put(key, 0L);
+ try {
+ Job job = Job.getInstance();
+ Tuple2<String, Long> tuple = new Tuple2<>();
+ tuple.setFields(key, value);
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
+
+ hadoopOutputFormat.recordWriter = new DummyRecordWriter();
+ hadoopOutputFormat.writeRecord(tuple);
+
+ Long expected = map.get(key);
+ assertEquals(expected, value);
+ } catch (IOException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testOpen() {
+ OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
+ try {
+ Job job = Job.getInstance();
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
+
+ hadoopOutputFormat.recordWriter = new DummyRecordWriter();
+ hadoopOutputFormat.open(1, 4);
+ } catch (IOException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testClose() {
+ OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
+ try {
+ Job job = Job.getInstance();
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
+
+ hadoopOutputFormat.recordWriter = new DummyRecordWriter();
+
+ final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+ Mockito.when(outputCommitter.needsTaskCommit(Mockito.any(TaskAttemptContext.class))).thenReturn(true);
+ Mockito.doNothing().when(outputCommitter).commitTask(Mockito.any(TaskAttemptContext.class));
+ hadoopOutputFormat.outputCommitter = outputCommitter;
+ hadoopOutputFormat.configuration = new Configuration();
+ hadoopOutputFormat.configuration.set("mapred.output.dir", PATH);
+
+ hadoopOutputFormat.close();
+ } catch (IOException e) {
+ fail();
+ }
+ }
+
+
+ class DummyRecordWriter extends RecordWriter<String, Long> {
+ @Override
+ public void write(String key, Long value) throws IOException, InterruptedException {
+ map.put(key, value);
+ }
+
+ @Override
+ public void close(TaskAttemptContext context) throws IOException, InterruptedException {
+
+ }
+ }
+
+ class DummyOutputFormat extends OutputFormat<String, Long> {
+ @Override
+ public RecordWriter<String, Long> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
+
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
+ final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+ Mockito.doNothing().when(outputCommitter).setupJob(Mockito.any(JobContext.class));
+
+ return outputCommitter;
+ }
+ }
+}
[3/3] flink git commit: [FLINK-2444] Add tests for HadoopInputFormats
Posted by fh...@apache.org.
[FLINK-2444] Add tests for HadoopInputFormats
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1ca4f36d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1ca4f36d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1ca4f36d
Branch: refs/heads/release-1.0
Commit: 1ca4f36d047cdefe04e12554a7df493aba681be3
Parents: da989c4
Author: Martin Liesenberg <ma...@gmail.com>
Authored: Fri Feb 12 00:18:06 2016 +0100
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Mar 23 17:37:31 2016 +0100
----------------------------------------------------------------------
.../hadoop/mapred/HadoopInputFormatBase.java | 4 +-
.../hadoop/mapreduce/HadoopInputFormat.java | 10 +-
.../hadoop/mapreduce/HadoopInputFormatBase.java | 12 +-
.../hadoop/mapred/HadoopInputFormatTest.java | 275 +++++++++++++++++--
.../hadoop/mapreduce/HadoopInputFormatTest.java | 202 ++++++++++++--
5 files changed, 443 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
index d80c80a..ef9999f 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatBase.java
@@ -148,7 +148,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
throws IOException {
org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
- for(int i=0;i<splitArray.length;i++){
+ for (int i=0; i<splitArray.length; i++) {
hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
}
return hiSplit;
@@ -177,7 +177,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
@Override
public boolean reachedEnd() throws IOException {
- if(!fetched) {
+ if (!fetched) {
fetchNext();
}
return !hasNext;
http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
index dadb954..abbcd25 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormat.java
@@ -18,15 +18,15 @@
package org.apache.flink.api.java.hadoop.mapreduce;
-import java.io.IOException;
-
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.hadoop.mapreduce.Job;
+
+import java.io.IOException;
/**
* InputFormat implementation allowing to use Hadoop (mapreduce) InputFormats with Flink.
@@ -49,10 +49,10 @@ public class HadoopInputFormat<K, V> extends HadoopInputFormatBase<K, V, Tuple2<
@Override
public Tuple2<K, V> nextRecord(Tuple2<K, V> record) throws IOException {
- if(!this.fetched) {
+ if (!this.fetched) {
fetchNext();
}
- if(!this.hasNext) {
+ if (!this.hasNext) {
return null;
}
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
index 4829ce7..73b11eb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatBase.java
@@ -111,7 +111,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
@Override
public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
// only gather base statistics for FileInputFormats
- if(!(mapreduceInputFormat instanceof FileInputFormat)) {
+ if (!(mapreduceInputFormat instanceof FileInputFormat)) {
return null;
}
@@ -158,7 +158,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
jobContext.getCredentials().addAll(this.credentials);
Credentials currentUserCreds = getCredentialsFromUGI(UserGroupInformation.getCurrentUser());
- if(currentUserCreds != null) {
+ if (currentUserCreds != null) {
jobContext.getCredentials().addAll(currentUserCreds);
}
@@ -170,7 +170,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
}
HadoopInputSplit[] hadoopInputSplits = new HadoopInputSplit[splits.size()];
- for(int i = 0; i < hadoopInputSplits.length; i++){
+ for (int i = 0; i < hadoopInputSplits.length; i++) {
hadoopInputSplits[i] = new HadoopInputSplit(i, splits.get(i), jobContext);
}
return hadoopInputSplits;
@@ -208,7 +208,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
@Override
public boolean reachedEnd() throws IOException {
- if(!this.fetched) {
+ if (!this.fetched) {
fetchNext();
}
return !this.hasNext;
@@ -243,7 +243,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
long latestModTime = 0L;
// get the file info and check whether the cached statistics are still valid.
- for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
+ for (org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
final Path filePath = new Path(hadoopPath.toUri());
final FileSystem fs = FileSystem.get(filePath.toUri());
@@ -308,7 +308,7 @@ public abstract class HadoopInputFormatBase<K, V, T> extends HadoopInputFormatCo
org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
configuration.readFields(in);
- if(this.configuration == null) {
+ if (this.configuration == null) {
this.configuration = configuration;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
index 63f8df4..3b8d227 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopInputFormatTest.java
@@ -20,21 +20,168 @@ package org.apache.flink.api.java.hadoop.mapred;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.hadoop.mapred.HadoopInputFormat;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobConfigurable;
+import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.junit.Test;
import java.io.IOException;
-import static org.junit.Assert.fail;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class HadoopInputFormatTest {
+ @Test
+ public void testConfigureWithConfigurableInstance() {
+ ConfigurableDummyInputFormat inputFormat = mock(ConfigurableDummyInputFormat.class);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+ verify(inputFormat, times(1)).setConf(any(JobConf.class));
+
+ hadoopInputFormat.configure(new org.apache.flink.configuration.Configuration());
+ verify(inputFormat, times(2)).setConf(any(JobConf.class));
+ }
+
+ @Test
+ public void testConfigureWithJobConfigurableInstance() {
+ JobConfigurableDummyInputFormat inputFormat = mock(JobConfigurableDummyInputFormat.class);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+ verify(inputFormat, times(1)).configure(any(JobConf.class));
+
+ hadoopInputFormat.configure(new org.apache.flink.configuration.Configuration());
+ verify(inputFormat, times(2)).configure(any(JobConf.class));
+ }
+
+ @Test
+ public void testOpenClose() throws Exception {
+ DummyRecordReader recordReader = mock(DummyRecordReader.class);
+ DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+ when(inputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+ hadoopInputFormat.open(getHadoopInputSplit());
+
+ verify(inputFormat, times(1)).getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class));
+ verify(recordReader, times(1)).createKey();
+ verify(recordReader, times(1)).createValue();
+
+ assertThat(hadoopInputFormat.fetched, is(false));
+
+ hadoopInputFormat.close();
+ verify(recordReader, times(1)).close();
+ }
+
+ @Test
+ public void testOpenWithConfigurableReader() throws Exception {
+ ConfigurableDummyRecordReader recordReader = mock(ConfigurableDummyRecordReader.class);
+ DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+ when(inputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+ hadoopInputFormat.open(getHadoopInputSplit());
+
+ verify(inputFormat, times(1)).getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class));
+ verify(recordReader, times(1)).setConf(any(JobConf.class));
+ verify(recordReader, times(1)).createKey();
+ verify(recordReader, times(1)).createValue();
+
+ assertThat(hadoopInputFormat.fetched, is(false));
+
+ }
+
+ @Test
+ public void testCreateInputSplits() throws Exception {
+
+ FileSplit[] result = new FileSplit[1];
+ result[0] = getFileSplit();
+ DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+ when(inputFormat.getSplits(any(JobConf.class), anyInt())).thenReturn(result);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+ hadoopInputFormat.createInputSplits(2);
+
+ verify(inputFormat, times(1)).getSplits(any(JobConf.class), anyInt());
+ }
+
+ @Test
+ public void testReachedEndWithElementsRemaining() throws IOException {
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(new DummyInputFormat(), String.class, Long.class, new JobConf());
+ hadoopInputFormat.fetched = true;
+ hadoopInputFormat.hasNext = true;
+
+ assertThat(hadoopInputFormat.reachedEnd(), is(false));
+ }
+
+ @Test
+ public void testReachedEndWithNoElementsRemaining() throws IOException {
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(new DummyInputFormat(), String.class, Long.class, new JobConf());
+ hadoopInputFormat.fetched = true;
+ hadoopInputFormat.hasNext = false;
+
+ assertThat(hadoopInputFormat.reachedEnd(), is(true));
+ }
+
+ @Test
+ public void testFetchNext() throws IOException {
+ DummyRecordReader recordReader = mock(DummyRecordReader.class);
+ when(recordReader.next(anyString(), anyLong())).thenReturn(true);
+
+ DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+ when(inputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat, String.class, Long.class, new JobConf());
+ hadoopInputFormat.open(getHadoopInputSplit());
+ hadoopInputFormat.fetchNext();
+
+ verify(recordReader, times(1)).next(anyString(), anyLong());
+ assertThat(hadoopInputFormat.hasNext, is(true));
+ assertThat(hadoopInputFormat.fetched, is(true));
+ }
+
+ @Test
+ public void checkTypeInformation() throws Exception {
+ HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<>(
+ new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, new JobConf());
+
+ TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
+ TypeInformation<Tuple2<Void,Long>> expectedType = new TupleTypeInfo<>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
+
+ assertThat(tupleType.isTupleType(), is(true));
+ assertThat(tupleType, is(equalTo(expectedType)));
+ }
+
+ private HadoopInputSplit getHadoopInputSplit() {
+ return new HadoopInputSplit(1, getFileSplit(), new JobConf());
+ }
+
+ private FileSplit getFileSplit() {
+ return new FileSplit(new Path("path"), 1, 2, new String[]{});
+ }
+
public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
public DummyVoidKeyInputFormat() {}
@@ -44,27 +191,107 @@ public class HadoopInputFormatTest {
return null;
}
}
-
- @Test
- public void checkTypeInformation() {
- try {
- HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>(
- new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, new JobConf());
-
- TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
- TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
-
- if(tupleType.isTupleType()) {
- if(!((TupleTypeInfo<?>)tupleType).equals(testTupleType)) {
- fail("Tuple type information was not set correctly!");
- }
- } else {
- fail("Type information was not set to tuple type information!");
- }
-
- }
- catch (Exception ex) {
- fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
+
+ public class DummyRecordReader implements RecordReader<String, Long> {
+
+ @Override
+ public float getProgress() throws IOException {
+ return 0;
+ }
+
+ @Override
+ public boolean next(String s, Long aLong) throws IOException {
+ return false;
}
+
+ @Override
+ public String createKey() {
+ return null;
+ }
+
+ @Override
+ public Long createValue() {
+ return null;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ }
+
+ public class ConfigurableDummyRecordReader implements RecordReader<String, Long>, Configurable {
+
+ @Override
+ public void setConf(Configuration configuration) {}
+
+ @Override
+ public Configuration getConf() {
+ return null;
+ }
+
+ @Override
+ public boolean next(String s, Long aLong) throws IOException {
+ return false;
+ }
+
+ @Override
+ public String createKey() {
+ return null;
+ }
+
+ @Override
+ public Long createValue() {
+ return null;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ return 0;
+ }
+ }
+
+ public class DummyInputFormat implements InputFormat<String, Long> {
+
+ @Override
+ public InputSplit[] getSplits(JobConf jobConf, int i) throws IOException {
+ return new InputSplit[0];
+ }
+
+ @Override
+ public RecordReader<String, Long> getRecordReader(InputSplit inputSplit, JobConf jobConf, Reporter reporter) throws IOException {
+ return null;
+ }
+ }
+
+ public class ConfigurableDummyInputFormat extends DummyInputFormat implements Configurable {
+ @Override
+ public void setConf(Configuration configuration) {}
+
+ @Override
+ public Configuration getConf() {
+ return null;
+ }
+ }
+
+ public class JobConfigurableDummyInputFormat extends DummyInputFormat implements JobConfigurable {
+
+ @Override
+ public void configure(JobConf jobConf) {}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1ca4f36d/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
index 919a286..d6ec484 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopInputFormatTest.java
@@ -20,22 +20,146 @@ package org.apache.flink.api.java.hadoop.mapreduce;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.hadoop.mapreduce.wrapper.HadoopInputSplit;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import java.io.IOException;
+import java.util.List;
-import static org.junit.Assert.fail;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class HadoopInputFormatTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testConfigure() throws Exception {
+
+ ConfigurableDummyInputFormat inputFormat = mock(ConfigurableDummyInputFormat.class);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(inputFormat, Job.getInstance(), null);
+ hadoopInputFormat.configure(new org.apache.flink.configuration.Configuration());
+
+ verify(inputFormat, times(1)).setConf(any(Configuration.class));
+ }
+
+ @Test
+ public void testCreateInputSplits() throws Exception {
+ DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(inputFormat, Job.getInstance(), null);
+ hadoopInputFormat.createInputSplits(2);
+
+ verify(inputFormat, times(1)).getSplits(any(JobContext.class));
+ }
+
+ @Test
+ public void testOpen() throws Exception {
+ DummyInputFormat inputFormat = mock(DummyInputFormat.class);
+ when(inputFormat.createRecordReader(any(InputSplit.class), any(TaskAttemptContext.class))).thenReturn(new DummyRecordReader());
+ HadoopInputSplit inputSplit = mock(HadoopInputSplit.class);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(inputFormat, Job.getInstance(), null);
+ hadoopInputFormat.open(inputSplit);
+
+ verify(inputFormat, times(1)).createRecordReader(any(InputSplit.class), any(TaskAttemptContext.class));
+ assertThat(hadoopInputFormat.fetched, is(false));
+ }
+
+ @Test
+ public void testClose() throws Exception {
+
+ DummyRecordReader recordReader = mock(DummyRecordReader.class);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+ hadoopInputFormat.close();
+
+ verify(recordReader, times(1)).close();
+ }
+
+ @Test
+ public void testFetchNextInitialState() throws Exception {
+ DummyRecordReader recordReader = new DummyRecordReader();
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+ hadoopInputFormat.fetchNext();
+
+ assertThat(hadoopInputFormat.fetched, is(true));
+ assertThat(hadoopInputFormat.hasNext, is(false));
+ }
+
+ @Test
+ public void testFetchNextRecordReaderHasNewValue() throws Exception {
+
+ DummyRecordReader recordReader = mock(DummyRecordReader.class);
+ when(recordReader.nextKeyValue()).thenReturn(true);
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+ hadoopInputFormat.fetchNext();
+
+ assertThat(hadoopInputFormat.fetched, is(true));
+ assertThat(hadoopInputFormat.hasNext, is(true));
+ }
+
+ @Test
+ public void testFetchNextRecordReaderThrowsException() throws Exception {
+
+ DummyRecordReader recordReader = mock(DummyRecordReader.class);
+ when(recordReader.nextKeyValue()).thenThrow(new InterruptedException());
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = setupHadoopInputFormat(new DummyInputFormat(), Job.getInstance(), recordReader);
+
+ exception.expect(IOException.class);
+ hadoopInputFormat.fetchNext();
+
+ assertThat(hadoopInputFormat.hasNext, is(true));
+ }
+
+ @Test
+ public void checkTypeInformation() throws Exception {
+
+ HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<>(
+ new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, Job.getInstance());
+
+ TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
+ TypeInformation<Tuple2<Void,Long>> expectedType = new TupleTypeInfo<>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
+
+ assertThat(tupleType.isTupleType(), is(true));
+ assertThat(tupleType, is(equalTo(expectedType)));
+ }
+
+
+ private HadoopInputFormat<String, Long> setupHadoopInputFormat(InputFormat<String, Long> inputFormat, Job job,
+ RecordReader<String, Long> recordReader) {
+
+ HadoopInputFormat<String, Long> hadoopInputFormat = new HadoopInputFormat<>(inputFormat,
+ String.class, Long.class, job);
+ hadoopInputFormat.recordReader = recordReader;
+
+ return hadoopInputFormat;
+ }
+
public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
public DummyVoidKeyInputFormat() {}
@@ -45,29 +169,61 @@ public class HadoopInputFormatTest {
return null;
}
}
-
- @Test
- public void checkTypeInformation() {
- try {
- // Set up the Hadoop Input Format
- Job job = Job.getInstance();
- HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>(
- new DummyVoidKeyInputFormat<Long>(), Void.class, Long.class, job);
-
- TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
- TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
-
- if(tupleType.isTupleType()) {
- if(!((TupleTypeInfo<?>)tupleType).equals(testTupleType)) {
- fail("Tuple type information was not set correctly!");
- }
- } else {
- fail("Type information was not set to tuple type information!");
- }
+
+ public class DummyRecordReader extends RecordReader<String, Long> {
+
+ @Override
+ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
}
- catch (Exception ex) {
- fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ return false;
+ }
+
+ @Override
+ public String getCurrentKey() throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
+ public Long getCurrentValue() throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ }
+
+ public class DummyInputFormat extends InputFormat<String, Long> {
+
+ @Override
+ public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
+ public RecordReader<String, Long> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ return new DummyRecordReader();
+ }
+ }
+
+ public class ConfigurableDummyInputFormat extends DummyInputFormat implements Configurable {
+
+ @Override
+ public void setConf(Configuration configuration) {}
+
+ @Override
+ public Configuration getConf() {
+ return null;
}
}
}
[2/3] flink git commit: [FLINK-2445] Improve HadoopOutputFormatTests
Posted by fh...@apache.org.
[FLINK-2445] Improve HadoopOutputFormatTests
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/da989c42
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/da989c42
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/da989c42
Branch: refs/heads/release-1.0
Commit: da989c423306212646f456d410562d9ec3779b24
Parents: edf343a
Author: Martin Liesenberg <ma...@gmail.com>
Authored: Thu Feb 11 15:59:01 2016 +0100
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Mar 23 17:37:15 2016 +0100
----------------------------------------------------------------------
.../hadoop/mapred/HadoopOutputFormatTest.java | 229 ++++++++++++++++
.../mapreduce/HadoopOutputFormatTest.java | 265 ++++++++++++-------
2 files changed, 395 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/da989c42/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java
new file mode 100644
index 0000000..409feb4
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.hadoop.mapred;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobConfigurable;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.util.Progressable;
+import org.junit.Test;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class HadoopOutputFormatTest {
+
+ @Test
+ public void testOpen() throws Exception {
+
+ OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+ DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+ JobConf jobConf = spy(new JobConf());
+ when(jobConf.getOutputCommitter()).thenReturn(outputCommitter);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+ outputFormat.open(1, 1);
+
+ verify(jobConf, times(2)).getOutputCommitter();
+ verify(outputCommitter, times(1)).setupJob(any(JobContext.class));
+ verify(dummyOutputFormat, times(1)).getRecordWriter(any(FileSystem.class), any(JobConf.class), anyString(), any(Progressable.class));
+ }
+
+ @Test
+ public void testConfigureWithConfigurable() {
+ ConfigurableDummyOutputFormat dummyOutputFormat = mock(ConfigurableDummyOutputFormat.class);
+ JobConf jobConf = mock(JobConf.class);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+ outputFormat.configure(Matchers.<org.apache.flink.configuration.Configuration>any());
+
+ verify(dummyOutputFormat, times(1)).setConf(any(Configuration.class));
+ }
+
+ @Test
+ public void testConfigureWithJobConfigurable() {
+ JobConfigurableDummyOutputFormat dummyOutputFormat = mock(JobConfigurableDummyOutputFormat.class);
+ JobConf jobConf = mock(JobConf.class);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+ outputFormat.configure(Matchers.<org.apache.flink.configuration.Configuration>any());
+
+ verify(dummyOutputFormat, times(1)).configure(any(JobConf.class));
+ }
+
+ @Test
+ public void testCloseWithTaskCommit() throws Exception {
+ OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+ DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+ when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(true);
+ DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
+ JobConf jobConf = mock(JobConf.class);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+ outputFormat.recordWriter = recordWriter;
+ outputFormat.outputCommitter = outputCommitter;
+
+ outputFormat.close();
+
+ verify(recordWriter, times(1)).close(any(Reporter.class));
+ verify(outputCommitter, times(1)).commitTask(any(TaskAttemptContext.class));
+ }
+
+ @Test
+ public void testCloseWithoutTaskCommit() throws Exception {
+ OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+ DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+ when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(false);
+ DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
+ JobConf jobConf = mock(JobConf.class);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+ outputFormat.recordWriter = recordWriter;
+ outputFormat.outputCommitter = outputCommitter;
+
+ outputFormat.close();
+
+ verify(recordWriter, times(1)).close(any(Reporter.class));
+ verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
+ }
+
+ @Test
+ public void testWriteRecord() throws Exception {
+ OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+ DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
+ JobConf jobConf = mock(JobConf.class);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+ outputFormat.recordWriter = recordWriter;
+
+ outputFormat.writeRecord(new Tuple2<>("key", 1L));
+
+ verify(recordWriter, times(1)).write(anyString(), anyLong());
+ }
+
+ @Test
+ public void testFinalizeGlobal() throws Exception {
+ OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+ DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
+ JobConf jobConf = spy(new JobConf());
+ when(jobConf.getOutputCommitter()).thenReturn(outputCommitter);
+
+ HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
+
+ outputFormat.finalizeGlobal(1);
+
+ verify(outputCommitter, times(1)).commitJob(any(JobContext.class));
+ }
+
+ public class DummyOutputFormat implements OutputFormat<String, Long> {
+
+ @Override
+ public RecordWriter<String, Long> getRecordWriter(FileSystem fileSystem, JobConf jobConf, String s, Progressable progressable) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void checkOutputSpecs(FileSystem fileSystem, JobConf jobConf) throws IOException {
+
+ }
+ }
+
+ public class ConfigurableDummyOutputFormat extends DummyOutputFormat implements Configurable {
+
+ @Override
+ public void setConf(Configuration configuration) {
+
+ }
+
+ @Override
+ public Configuration getConf() {
+ return null;
+ }
+ }
+
+ public class JobConfigurableDummyOutputFormat extends DummyOutputFormat implements JobConfigurable {
+
+ @Override
+ public void configure(JobConf jobConf) {
+
+ }
+ }
+
+ public class DummyOutputCommitter extends OutputCommitter {
+
+ @Override
+ public void setupJob(JobContext jobContext) throws IOException {
+
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
+ return false;
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+ }
+ }
+
+ public class DummyRecordWriter implements RecordWriter<String, Long> {
+
+ @Override
+ public void write(String s, Long aLong) throws IOException {
+
+ }
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/da989c42/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
index 215ca95..7b126fc 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatTest.java
@@ -19,112 +19,179 @@
package org.apache.flink.api.java.hadoop.mapreduce;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class HadoopOutputFormatTest {
- private static final String PATH = "an/ignored/file/";
- private Map<String, Long> map;
-
- @Test
- public void testWriteRecord() {
- OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
- String key = "Test";
- Long value = 1L;
- map = new HashMap<>();
- map.put(key, 0L);
- try {
- Job job = Job.getInstance();
- Tuple2<String, Long> tuple = new Tuple2<>();
- tuple.setFields(key, value);
- HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
-
- hadoopOutputFormat.recordWriter = new DummyRecordWriter();
- hadoopOutputFormat.writeRecord(tuple);
-
- Long expected = map.get(key);
- assertEquals(expected, value);
- } catch (IOException e) {
- fail();
- }
- }
-
- @Test
- public void testOpen() {
- OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
- try {
- Job job = Job.getInstance();
- HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
-
- hadoopOutputFormat.recordWriter = new DummyRecordWriter();
- hadoopOutputFormat.open(1, 4);
- } catch (IOException e) {
- fail();
- }
- }
-
- @Test
- public void testClose() {
- OutputFormat<String, Long> dummyOutputFormat = new DummyOutputFormat();
- try {
- Job job = Job.getInstance();
- HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(dummyOutputFormat, job);
-
- hadoopOutputFormat.recordWriter = new DummyRecordWriter();
-
- final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
- Mockito.when(outputCommitter.needsTaskCommit(Mockito.any(TaskAttemptContext.class))).thenReturn(true);
- Mockito.doNothing().when(outputCommitter).commitTask(Mockito.any(TaskAttemptContext.class));
- hadoopOutputFormat.outputCommitter = outputCommitter;
- hadoopOutputFormat.configuration = new Configuration();
- hadoopOutputFormat.configuration.set("mapred.output.dir", PATH);
-
- hadoopOutputFormat.close();
- } catch (IOException e) {
- fail();
- }
- }
-
-
- class DummyRecordWriter extends RecordWriter<String, Long> {
- @Override
- public void write(String key, Long value) throws IOException, InterruptedException {
- map.put(key, value);
- }
-
- @Override
- public void close(TaskAttemptContext context) throws IOException, InterruptedException {
-
- }
- }
-
- class DummyOutputFormat extends OutputFormat<String, Long> {
- @Override
- public RecordWriter<String, Long> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
- return null;
- }
-
- @Override
- public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
-
- }
-
- @Override
- public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
- final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
- Mockito.doNothing().when(outputCommitter).setupJob(Mockito.any(JobContext.class));
-
- return outputCommitter;
- }
- }
+ private static final String MAPRED_OUTPUT_PATH = "an/ignored/file/";
+ private static final String MAPRED_OUTPUT_DIR_KEY = "mapred.output.dir";
+
+ @Test
+ public void testWriteRecord() throws Exception {
+
+ RecordWriter<String, Long> recordWriter = mock(DummyRecordWriter.class);
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+ Job.getInstance(), recordWriter, null, new Configuration());
+
+ hadoopOutputFormat.writeRecord(new Tuple2<String, Long>());
+
+ verify(recordWriter, times(1)).write(anyString(), anyLong());
+ }
+
+ @Test
+ public void testOpen() throws Exception {
+
+
+ OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
+ OutputCommitter outputCommitter = setupOutputCommitter(true);
+ when(dummyOutputFormat.getOutputCommitter(any(TaskAttemptContext.class))).thenReturn(outputCommitter);
+
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(dummyOutputFormat,
+ Job.getInstance(), new DummyRecordWriter(), setupOutputCommitter(true), new Configuration());
+
+ hadoopOutputFormat.open(1, 4);
+
+ verify(hadoopOutputFormat.outputCommitter, times(1)).setupJob(any(JobContext.class));
+ verify(hadoopOutputFormat.mapreduceOutputFormat, times(1)).getRecordWriter(any(TaskAttemptContext.class));
+ }
+
+ @Test
+ public void testCloseWithNeedsTaskCommitTrue() throws Exception {
+
+ RecordWriter<String, Long> recordWriter = Mockito.mock(DummyRecordWriter.class);
+ OutputCommitter outputCommitter = setupOutputCommitter(true);
+
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+ Job.getInstance(), recordWriter, outputCommitter, new Configuration());
+
+ hadoopOutputFormat.close();
+
+ verify(outputCommitter, times(1)).commitTask(any(TaskAttemptContext.class));
+ verify(recordWriter, times(1)).close(any(TaskAttemptContext.class));
+ }
+
+ @Test
+ public void testCloseWithNeedsTaskCommitFalse() throws Exception {
+
+ RecordWriter<String, Long> recordWriter = Mockito.mock(DummyRecordWriter.class);
+ OutputCommitter outputCommitter = setupOutputCommitter(false);
+
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+ Job.getInstance(), recordWriter, outputCommitter, new Configuration());
+
+ hadoopOutputFormat.close();
+
+ verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
+ verify(recordWriter, times(1)).close(any(TaskAttemptContext.class));
+ }
+
+ @Test
+ public void testConfigure() throws Exception {
+
+ ConfigurableDummyOutputFormat outputFormat = mock(ConfigurableDummyOutputFormat.class);
+
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(outputFormat, Job.getInstance(),
+ null, null, new Configuration());
+
+ hadoopOutputFormat.configure(new org.apache.flink.configuration.Configuration());
+
+ verify(outputFormat, times(1)).setConf(any(Configuration.class));
+ }
+
+ @Test
+ public void testFinalizedGlobal() throws Exception {
+
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = setupHadoopOutputFormat(new DummyOutputFormat(),
+ Job.getInstance(), null, null, new Configuration());
+
+ hadoopOutputFormat.finalizeGlobal(1);
+
+ verify(hadoopOutputFormat.outputCommitter, times(1)).commitJob(any(JobContext.class));
+ }
+
+ private OutputCommitter setupOutputCommitter(boolean needsTaskCommit) throws IOException {
+ OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+ when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(needsTaskCommit);
+ doNothing().when(outputCommitter).commitTask(any(TaskAttemptContext.class));
+
+ return outputCommitter;
+ }
+
+ private HadoopOutputFormat<String, Long> setupHadoopOutputFormat(OutputFormat<String, Long> outputFormat,
+ Job job,
+ RecordWriter<String, Long> recordWriter,
+ OutputCommitter outputCommitter,
+ Configuration configuration) {
+
+ HadoopOutputFormat<String, Long> hadoopOutputFormat = new HadoopOutputFormat<>(outputFormat, job);
+ hadoopOutputFormat.recordWriter = recordWriter;
+ hadoopOutputFormat.outputCommitter = outputCommitter;
+ hadoopOutputFormat.configuration = configuration;
+ hadoopOutputFormat.configuration.set(MAPRED_OUTPUT_DIR_KEY, MAPRED_OUTPUT_PATH);
+
+ return hadoopOutputFormat;
+ }
+
+ class DummyRecordWriter extends RecordWriter<String, Long> {
+ @Override
+ public void write(String key, Long value) throws IOException, InterruptedException {
+ }
+
+ @Override
+ public void close(TaskAttemptContext context) throws IOException, InterruptedException {
+
+ }
+ }
+
+ class DummyOutputFormat extends OutputFormat<String, Long> {
+ @Override
+ public RecordWriter<String, Long> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
+
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
+ final OutputCommitter outputCommitter = Mockito.mock(OutputCommitter.class);
+ doNothing().when(outputCommitter).setupJob(any(JobContext.class));
+
+ return outputCommitter;
+ }
+ }
+
+ class ConfigurableDummyOutputFormat extends DummyOutputFormat implements Configurable {
+
+ @Override
+ public void setConf(Configuration configuration) {}
+
+ @Override
+ public Configuration getConf() {
+ return null;
+ }
+ }
}