You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2021/01/09 03:10:18 UTC
[orc] branch master updated: ORC-714: Remove MRUnit and its usage
(#612)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/master by this push:
new db62ea5 ORC-714: Remove MRUnit and its usage (#612)
db62ea5 is described below
commit db62ea5730797b046a3b7d5d1a3b026b08921209
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Fri Jan 8 19:10:07 2021 -0800
ORC-714: Remove MRUnit and its usage (#612)
### What changes were proposed in this pull request?
This PR aims to remove `mrunit` and its dependency and usage. This will unblock us toward new Java versions like Java 17.
```
[INFO] +- org.apache.mrunit:mrunit:jar:hadoop2:1.1.0:test
[INFO] | +- org.powermock:powermock-core:jar:1.5.1:test
[INFO] | | +- org.powermock:powermock-reflect:jar:1.5.1:test
[INFO] | | \- org.javassist:javassist:jar:3.18.0-GA:test
[INFO] | +- org.powermock:powermock-api-mockito:jar:1.5.1:test
[INFO] | | \- org.powermock:powermock-api-support:jar:1.5.1:test
[INFO] | \- org.powermock:powermock-module-junit4:jar:1.5.1:test
[INFO] | \- org.powermock:powermock-module-junit4-common:jar:1.5.1:test
```
### Why are the changes needed?
We are using the last version, `mrunit`, which is released at `2014/06/06` and is retired `2016/04/30`. We had better remove this and use Hadoop's minicluster or alternatives.
### How was this patch tested?
Pass the CIs because this is a removal of test coverage. ORC-715 will add a test coverage back.
---
java/mapreduce/pom.xml | 6 -
.../src/test/org/apache/orc/mapred/TestMrUnit.java | 223 ---------------------
.../test/org/apache/orc/mapreduce/TestMrUnit.java | 203 -------------------
java/pom.xml | 7 -
4 files changed, 439 deletions(-)
diff --git a/java/mapreduce/pom.xml b/java/mapreduce/pom.xml
index dc44c63..7351313 100644
--- a/java/mapreduce/pom.xml
+++ b/java/mapreduce/pom.xml
@@ -80,12 +80,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <scope>test</scope>
- <classifier>hadoop2</classifier>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestMrUnit.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestMrUnit.java
deleted file mode 100644
index cd11603..0000000
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestMrUnit.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.orc.mapred;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.serializer.Deserializer;
-import org.apache.hadoop.io.serializer.Serialization;
-import org.apache.hadoop.io.serializer.Serializer;
-import org.apache.hadoop.io.serializer.WritableSerialization;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mrunit.MapReduceDriver;
-import org.apache.orc.OrcConf;
-import org.apache.orc.TypeDescription;
-import org.junit.Test;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Iterator;
-
-public class TestMrUnit {
- JobConf conf = new JobConf();
-
- /**
- * Split the input struct into its two parts.
- */
- public static class MyMapper
- implements Mapper<NullWritable, OrcStruct, OrcKey, OrcValue> {
- private OrcKey keyWrapper = new OrcKey();
- private OrcValue valueWrapper = new OrcValue();
-
- @Override
- public void map(NullWritable key, OrcStruct value,
- OutputCollector<OrcKey, OrcValue> outputCollector,
- Reporter reporter) throws IOException {
- keyWrapper.key = value.getFieldValue(0);
- valueWrapper.value = value.getFieldValue(1);
- outputCollector.collect(keyWrapper, valueWrapper);
- }
-
- @Override
- public void close() throws IOException {
- // PASS
- }
-
- @Override
- public void configure(JobConf jobConf) {
- // PASS
- }
- }
-
- /**
- * Glue the key and values back together.
- */
- public static class MyReducer
- implements Reducer<OrcKey, OrcValue, NullWritable, OrcStruct> {
- private OrcStruct output = new OrcStruct(TypeDescription.fromString
- ("struct<first:struct<x:int,y:int>,second:struct<z:string>>"));
- private final NullWritable nada = NullWritable.get();
-
- @Override
- public void reduce(OrcKey key, Iterator<OrcValue> iterator,
- OutputCollector<NullWritable, OrcStruct> collector,
- Reporter reporter) throws IOException {
- output.setFieldValue(0, key.key);
- while (iterator.hasNext()) {
- OrcValue value = iterator.next();
- output.setFieldValue(1, value.value);
- collector.collect(nada, output);
- }
- }
-
- @Override
- public void close() throws IOException {
- // PASS
- }
-
- @Override
- public void configure(JobConf jobConf) {
- // PASS
- }
- }
-
- /**
- * This class is intended to support MRUnit's object copying for input and
- * output objects.
- *
- * Real mapreduce contexts should NEVER use this class.
- *
- * The type string is serialized before each value.
- */
- public static class OrcStructSerialization
- implements Serialization<OrcStruct> {
-
- @Override
- public boolean accept(Class<?> cls) {
- return OrcStruct.class.isAssignableFrom(cls);
- }
-
- @Override
- public Serializer<OrcStruct> getSerializer(Class<OrcStruct> aClass) {
- return new Serializer<OrcStruct>() {
- DataOutputStream dataOut;
-
- public void open(OutputStream out) {
- if(out instanceof DataOutputStream) {
- dataOut = (DataOutputStream)out;
- } else {
- dataOut = new DataOutputStream(out);
- }
- }
-
- public void serialize(OrcStruct w) throws IOException {
- Text.writeString(dataOut, w.getSchema().toString());
- w.write(dataOut);
- }
-
- public void close() throws IOException {
- dataOut.close();
- }
- };
- }
-
- @Override
- public Deserializer<OrcStruct> getDeserializer(Class<OrcStruct> aClass) {
- return new Deserializer<OrcStruct>() {
- DataInputStream input;
-
- @Override
- public void open(InputStream inputStream) throws IOException {
- if(inputStream instanceof DataInputStream) {
- input = (DataInputStream)inputStream;
- } else {
- input = new DataInputStream(inputStream);
- }
- }
-
- @Override
- public OrcStruct deserialize(OrcStruct orcStruct) throws IOException {
- String typeStr = Text.readString(input);
- OrcStruct result = new OrcStruct(TypeDescription.fromString(typeStr));
- result.readFields(input);
- return result;
- }
-
- @Override
- public void close() throws IOException {
- // PASS
- }
- };
- }
- }
-
- @Test
- public void testMapred() throws IOException {
- conf.set("io.serializations",
- OrcStructSerialization.class.getName() + "," +
- WritableSerialization.class.getName());
- OrcConf.MAPRED_SHUFFLE_KEY_SCHEMA.setString(conf, "struct<x:int,y:int>");
- OrcConf.MAPRED_SHUFFLE_VALUE_SCHEMA.setString(conf, "struct<z:string>");
- MyMapper mapper = new MyMapper();
- mapper.configure(conf);
- MyReducer reducer = new MyReducer();
- reducer.configure(conf);
- MapReduceDriver<NullWritable, OrcStruct,
- OrcKey, OrcValue,
- NullWritable, OrcStruct> driver =
- new MapReduceDriver<>(mapper, reducer);
- driver.setConfiguration(conf);
- NullWritable nada = NullWritable.get();
- OrcStruct input = (OrcStruct) OrcStruct.createValue(
- TypeDescription.fromString("struct<one:struct<x:int,y:int>,two:struct<z:string>>"));
- IntWritable x =
- (IntWritable) ((OrcStruct) input.getFieldValue(0)).getFieldValue(0);
- IntWritable y =
- (IntWritable) ((OrcStruct) input.getFieldValue(0)).getFieldValue(1);
- Text z = (Text) ((OrcStruct) input.getFieldValue(1)).getFieldValue(0);
-
- // generate the input stream
- for(int r=0; r < 20; ++r) {
- x.set(100 - (r / 4));
- y.set(r*2);
- z.set(Integer.toHexString(r));
- driver.withInput(nada, input);
- }
-
- // generate the expected outputs
- for(int g=4; g >= 0; --g) {
- x.set(100 - g);
- for(int i=0; i < 4; ++i) {
- int r = g * 4 + i;
- y.set(r * 2);
- z.set(Integer.toHexString(r));
- driver.withOutput(nada, input);
- }
- }
- driver.runTest();
- }
-}
diff --git a/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMrUnit.java b/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMrUnit.java
deleted file mode 100644
index 01208e1..0000000
--- a/java/mapreduce/src/test/org/apache/orc/mapreduce/TestMrUnit.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.orc.mapreduce;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.serializer.Deserializer;
-import org.apache.hadoop.io.serializer.Serialization;
-import org.apache.hadoop.io.serializer.Serializer;
-import org.apache.hadoop.io.serializer.WritableSerialization;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
-import org.apache.orc.OrcConf;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.mapred.OrcKey;
-import org.apache.orc.mapred.OrcStruct;
-import org.apache.orc.mapred.OrcValue;
-import org.junit.Test;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Iterator;
-
-public class TestMrUnit {
- JobConf conf = new JobConf();
-
- /**
- * Split the input struct into its two parts.
- */
- public static class MyMapper
- extends Mapper<NullWritable, OrcStruct, OrcKey, OrcValue> {
- private OrcKey keyWrapper = new OrcKey();
- private OrcValue valueWrapper = new OrcValue();
-
- @Override
- protected void map(NullWritable key,
- OrcStruct value,
- Context context
- ) throws IOException, InterruptedException {
- keyWrapper.key = value.getFieldValue(0);
- valueWrapper.value = value.getFieldValue(1);
- context.write(keyWrapper, valueWrapper);
- }
- }
-
- /**
- * Glue the key and values back together.
- */
- public static class MyReducer
- extends Reducer<OrcKey, OrcValue, NullWritable, OrcStruct> {
- private OrcStruct output = new OrcStruct(TypeDescription.fromString
- ("struct<first:struct<x:int,y:int>,second:struct<z:string>>"));
- private final NullWritable nada = NullWritable.get();
-
- @Override
- protected void reduce(OrcKey key,
- Iterable<OrcValue> values,
- Context context
- ) throws IOException, InterruptedException {
- output.setFieldValue(0, key.key);
- for(OrcValue value: values) {
- output.setFieldValue(1, value.value);
- context.write(nada, output);
- }
- }
- }
-
- /**
- * This class is intended to support MRUnit's object copying for input and
- * output objects.
- *
- * Real mapreduce contexts should NEVER use this class.
- *
- * The type string is serialized before each value.
- */
- public static class OrcStructSerialization
- implements Serialization<OrcStruct> {
-
- @Override
- public boolean accept(Class<?> cls) {
- return OrcStruct.class.isAssignableFrom(cls);
- }
-
- @Override
- public Serializer<OrcStruct> getSerializer(Class<OrcStruct> aClass) {
- return new Serializer<OrcStruct>() {
- DataOutputStream dataOut;
-
- public void open(OutputStream out) {
- if(out instanceof DataOutputStream) {
- dataOut = (DataOutputStream)out;
- } else {
- dataOut = new DataOutputStream(out);
- }
- }
-
- public void serialize(OrcStruct w) throws IOException {
- Text.writeString(dataOut, w.getSchema().toString());
- w.write(dataOut);
- }
-
- public void close() throws IOException {
- dataOut.close();
- }
- };
- }
-
- @Override
- public Deserializer<OrcStruct> getDeserializer(Class<OrcStruct> aClass) {
- return new Deserializer<OrcStruct>() {
- DataInputStream input;
-
- @Override
- public void open(InputStream inputStream) throws IOException {
- if(inputStream instanceof DataInputStream) {
- input = (DataInputStream)inputStream;
- } else {
- input = new DataInputStream(inputStream);
- }
- }
-
- @Override
- public OrcStruct deserialize(OrcStruct orcStruct) throws IOException {
- String typeStr = Text.readString(input);
- OrcStruct result = new OrcStruct(TypeDescription.fromString(typeStr));
- result.readFields(input);
- return result;
- }
-
- @Override
- public void close() throws IOException {
- // PASS
- }
- };
- }
- }
-
- @Test
- public void testMapred() throws IOException {
- conf.set("io.serializations",
- OrcStructSerialization.class.getName() + "," +
- WritableSerialization.class.getName());
- OrcConf.MAPRED_SHUFFLE_KEY_SCHEMA.setString(conf, "struct<x:int,y:int>");
- OrcConf.MAPRED_SHUFFLE_VALUE_SCHEMA.setString(conf, "struct<z:string>");
- MyMapper mapper = new MyMapper();
- MyReducer reducer = new MyReducer();
- MapReduceDriver<NullWritable, OrcStruct,
- OrcKey, OrcValue,
- NullWritable, OrcStruct> driver =
- new MapReduceDriver<>(mapper, reducer);
- driver.setConfiguration(conf);
- NullWritable nada = NullWritable.get();
- OrcStruct input = (OrcStruct) OrcStruct.createValue(
- TypeDescription.fromString("struct<one:struct<x:int,y:int>,two:struct<z:string>>"));
- IntWritable x =
- (IntWritable) ((OrcStruct) input.getFieldValue(0)).getFieldValue(0);
- IntWritable y =
- (IntWritable) ((OrcStruct) input.getFieldValue(0)).getFieldValue(1);
- Text z = (Text) ((OrcStruct) input.getFieldValue(1)).getFieldValue(0);
-
- // generate the input stream
- for(int r=0; r < 20; ++r) {
- x.set(100 - (r / 4));
- y.set(r*2);
- z.set(Integer.toHexString(r));
- driver.withInput(nada, input);
- }
-
- // generate the expected outputs
- for(int g=4; g >= 0; --g) {
- x.set(100 - g);
- for(int i=0; i < 4; ++i) {
- int r = g * 4 + i;
- y.set(r * 2);
- z.set(Integer.toHexString(r));
- driver.withOutput(nada, input);
- }
- }
- driver.runTest();
- }
-}
diff --git a/java/pom.xml b/java/pom.xml
index f4151f4..b7fa827 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -702,13 +702,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <version>1.1.0</version>
- <scope>test</scope>
- <classifier>hadoop2</classifier>
- </dependency>
- <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>1.3</version>