You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/13 05:54:11 UTC
[05/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into
standard directory layout. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
deleted file mode 100644
index 8a4aa6f..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.IntSumReducer;
-import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.TokenizerMapper;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CombinerTest {
- private FileSystem fs;
- private String inputpath;
- private String nativeoutputpath;
- private String hadoopoutputpath;
-
- @Test
- public void testWordCountCombiner() {
- try {
-
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
-
- final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
- commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
-
- final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
-
- nativejob.waitForCompletion(true);
-
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- normaljob.waitForCompletion(true);
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
- assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
- nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-
- } catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
- }
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration conf = new ScenarioConfiguration();
- conf.addcombinerConf();
-
- this.fs = FileSystem.get(conf);
-
- this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
-
- if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(
- conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000),
- Text.class.getName(),
- Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
- }
-
- this.nativeoutputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativewordcount";
- this.hadoopoutputpath = conf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalwordcount";
- }
-
- protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)
- throws Exception {
- final Configuration conf = new Configuration(inputConf);
- conf.set("fileoutputpath", outputpath);
- final FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputpath))) {
- fs.delete(new Path(outputpath));
- }
- fs.close();
- final Job job = new Job(conf, jobname);
- job.setJarByClass(WordCount.class);
- job.setMapperClass(TokenizerMapper.class);
- job.setCombinerClass(IntSumReducer.class);
- job.setReducerClass(IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputpath));
- FileOutputFormat.setOutputPath(job, new Path(outputpath));
- return job;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
deleted file mode 100644
index 50953e0..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Test;
-
-public class LargeKVCombinerTest {
-
- @Test
- public void testLargeValueCombiner(){
- final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final int deafult_KVSize_Maximum = 1 << 22; // 4M
- final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
- deafult_KVSize_Maximum);
- final String inputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/largeKV";
- final String nativeOutputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativeLargeKV";
- final String hadoopOutputPath = normalConf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
- TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalLargeKV";
- try {
- final FileSystem fs = FileSystem.get(normalConf);
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
-
- int max = i;
- int min = Math.max(i / 4, max - 10);
-
- System.out.println("===KV Size Test: min size: " + min + ", max size: " + max);
-
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- fs.delete(new Path(inputPath), true);
- new TestInputFile(normalConf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
- 1000000), IntWritable.class.getName(),
- Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
-
- final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
- final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
-
- nativejob.waitForCompletion(true);
- Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- normaljob.waitForCompletion(true);
- Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-
- final String reason = "LargeKVCombinerTest failed with, min size: " + min
- + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
-
- assertEquals(reason, true, compareRet);
-// assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter",
-// nativeReduceGroups.getValue(), normalReduceGroups.getValue());
- }
- fs.close();
- } catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
deleted file mode 100644
index 5691e02..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.junit.Before;
-import org.junit.Test;
-
-public class OldAPICombinerTest {
- private FileSystem fs;
- private String inputpath;
-
- @Test
- public void testWordCountCombinerWithOldAPI() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final String nativeoutput = nativeConf.get(TestConstants.OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH);
- final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
- RunningJob nativeRunning = JobClient.runJob(nativeJob);
-
- Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final String normaloutput = normalConf.get(TestConstants.OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH);
- final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
-
- RunningJob normalRunning = JobClient.runJob(normalJob);
- Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-
- assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue());
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration conf = new ScenarioConfiguration();
- conf.addcombinerConf();
- this.fs = FileSystem.get(conf);
- this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
- TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
-
- if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(conf.getInt("nativetask.combiner.wordcount.filesize", 1000000), Text.class.getName(),
- Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
- }
- }
-
- private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
- throws Exception {
- final JobConf jobConf = new JobConf(configuration);
- final FileSystem fs = FileSystem.get(configuration);
- if (fs.exists(new Path(output))) {
- fs.delete(new Path(output), true);
- }
- fs.close();
- jobConf.setJobName(name);
- jobConf.setOutputKeyClass(Text.class);
- jobConf.setOutputValueClass(IntWritable.class);
- jobConf.setMapperClass(WordCountWithOldAPI.TokenizerMapperWithOldAPI.class);
- jobConf.setCombinerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
- jobConf.setReducerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
-
- jobConf.setInputFormat(SequenceFileInputFormat.class);
- jobConf.setOutputFormat(TextOutputFormat.class);
-
- FileInputFormat.setInputPaths(jobConf, new Path(input));
- FileOutputFormat.setOutputPath(jobConf, new Path(output));
- return jobConf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
deleted file mode 100644
index 490b82b..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-
-public class WordCount {
-
- private static Log LOG = LogFactory.getLog(WordCount.class);
-
- public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
-
- private final static IntWritable one = new IntWritable(1);
- private final Text word = new Text();
-
- @Override
- public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
- final StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
- private final IntWritable result = new IntWritable();
-
- @Override
- public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
- InterruptedException {
- int sum = 0;
- for (final IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(key, result);
- }
- }
-
- public static void main(String[] args) throws Exception {
- final Configuration conf = new Configuration();
- final String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- if (otherArgs.length != 2) {
- System.err.println("Usage: wordcount <in> <out>");
- System.exit(2);
- }
- final Job job = new Job(conf, conf.get(MRJobConfig.JOB_NAME, "word count"));
- job.setJarByClass(WordCount.class);
- job.setMapperClass(TokenizerMapper.class);
- job.setCombinerClass(IntSumReducer.class);
- job.setReducerClass(IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
- FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
- System.exit(job.waitForCompletion(true) ? 0 : 1);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
deleted file mode 100644
index a11ea91..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-public class WordCountWithOldAPI {
-
- public static class TokenizerMapperWithOldAPI extends MapReduceBase implements
- Mapper<Object, Text, Text, IntWritable> {
- private final static IntWritable one = new IntWritable(1);
- private final Text word = new Text();
-
- @Override
- public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
- throws IOException {
- final StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- output.collect(word, one);
- }
- }
- }
-
- public static class IntSumReducerWithOldAPI extends MapReduceBase implements
- Reducer<Text, IntWritable, Text, IntWritable> {
- private final IntWritable result = new IntWritable();
-
- @Override
- public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- int sum = 0;
- while (values.hasNext()) {
- sum += values.next().get();
- }
- result.set(sum);
- output.collect(key, result);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
deleted file mode 100644
index 83c4794..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.compresstest;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class CompressMapper {
- public static final String inputFile = "./compress/input.txt";
- public static final String outputFileDir = "./compress/output/";
-
- public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
-
- @Override
- protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- context.write(key, value);
- }
- }
-
- public static Job getCompressJob(String jobname, Configuration conf) {
- Job job = null;
- try {
- job = new Job(conf, jobname + "-CompressMapperJob");
- job.setJarByClass(CompressMapper.class);
- job.setMapperClass(TextCompressMapper.class);
- job.setOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Text.class);
- final Path outputpath = new Path(outputFileDir + jobname);
- // if output file exists ,delete it
- final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
- if (hdfs.exists(outputpath)) {
- hdfs.delete(outputpath);
- }
- hdfs.close();
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputFile));
- FileOutputFormat.setOutputPath(job, outputpath);
- } catch (final Exception e) {
- e.printStackTrace();
- }
- return job;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
deleted file mode 100644
index 0406375..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.compresstest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CompressTest {
-
- @Test
- public void testSnappyCompress() throws Exception {
- final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
- conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
- final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
- job.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
- CompressMapper.outputFileDir + "hadoopsnappy");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testGzipCompress() throws Exception {
- final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
- conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
- final Job job = CompressMapper.getCompressJob("nativegzip", conf);
- job.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
- CompressMapper.outputFileDir + "hadoopgzip");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testBzip2Compress() throws Exception {
- final Configuration nativeconf = ScenarioConfiguration.getNativeConfiguration();
- nativeconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
- final Job nativejob = CompressMapper.getCompressJob("nativebzip2", nativeconf);
- nativejob.waitForCompletion(true);
-
- final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
- hadoopconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
- final Job hadoopjob = CompressMapper.getCompressJob("hadoopbzip2", hadoopconf);
- hadoopjob.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativebzip2",
- CompressMapper.outputFileDir + "hadoopbzip2");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testLz4Compress() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
- final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
- nativeJob.waitForCompletion(true);
-
- final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
- hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
- final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
- hadoopJob.waitForCompletion(true);
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
- CompressMapper.outputFileDir + "hadooplz4");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Test
- public void testDefaultCompress() throws Exception {
- final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
- final Job nativeJob = CompressMapper.getCompressJob("nativedefault", nativeConf);
- nativeJob.waitForCompletion(true);
-
- final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
- hadoopConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
- final Job hadoopJob = CompressMapper.getCompressJob("hadoopdefault", hadoopConf);
- hadoopJob.waitForCompletion(true);
- final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativedefault",
- CompressMapper.outputFileDir + "hadoopdefault");
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration conf = new ScenarioConfiguration();
- final FileSystem fs = FileSystem.get(conf);
- final Path path = new Path(CompressMapper.inputFile);
- fs.delete(path);
- if (!fs.exists(path)) {
- new TestInputFile(ScenarioConfiguration.getNormalConfiguration().getInt(
- TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
- Text.class.getName(), Text.class.getName(), conf)
- .createSequenceTestFile(CompressMapper.inputFile);
-
- }
- fs.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
deleted file mode 100644
index ce23780..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Reducer;
-
-public class HashSumReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, IntWritable> {
-
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(os);
-
- @Override
- public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
- int hashSum = 0;
- for (final VTYPE val : values) {
- if (val instanceof Writable) {
- os.reset();
- ((Writable) val).write(dos);
- final int hash = Arrays.hashCode(os.toByteArray());
- hashSum += hash;
- }
- }
-
- context.write(key, new IntWritable(hashSum));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
deleted file mode 100644
index 6d683f8..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.IOException;
-import java.util.zip.CRC32;
-
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class KVJob {
- public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
- public static final String OUTPUTPATH = "nativetask.kvtest.outputfile.path";
- Job job = null;
-
- public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {
- @Override
- public void map(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
- context.write(key, value);
- }
- }
-
- public static class KVMReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
- public void reduce(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
- context.write(key, value);
- }
- }
-
- public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
-
- @Override
- public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
- long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function
- final CRC32 crc32 = new CRC32();
- for (final VTYPE val : values) {
- crc32.reset();
- crc32.update(BytesFactory.toBytes(val));
- resultlong += crc32.getValue();
- }
- final VTYPE V = null;
- context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName()));
- }
- }
-
- public KVJob(String jobname, Configuration conf, Class<?> keyclass, Class<?> valueclass, String inputpath,
- String outputpath) throws Exception {
- job = new Job(conf, jobname);
- job.setJarByClass(KVJob.class);
- job.setMapperClass(KVJob.ValueMapper.class);
- job.setOutputKeyClass(keyclass);
- job.setMapOutputValueClass(valueclass);
-
- if (conf.get(TestConstants.NATIVETASK_KVTEST_CREATEFILE).equals("true")) {
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(inputpath), true);
- fs.close();
- final TestInputFile testfile = new TestInputFile(Integer.valueOf(conf.get(
- TestConstants.FILESIZE_KEY, "1000")),
- keyclass.getName(), valueclass.getName(), conf);
- testfile.createSequenceTestFile(inputpath);
-
- }
- job.setInputFormatClass(SequenceFileInputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputpath));
- FileOutputFormat.setOutputPath(job, new Path(outputpath));
- }
-
- public void runJob() throws Exception {
-
- job.waitForCompletion(true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
deleted file mode 100644
index 1e08854..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class KVTest {
- private static Class<?>[] keyclasses = null;
- private static Class<?>[] valueclasses = null;
- private static String[] keyclassNames = null;
- private static String[] valueclassNames = null;
-
- private static Configuration nativekvtestconf = ScenarioConfiguration.getNativeConfiguration();
- private static Configuration hadoopkvtestconf = ScenarioConfiguration.getNormalConfiguration();
- static {
- nativekvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
- hadoopkvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
- }
-
- @Parameters(name = "key:{0}\nvalue:{1}")
- public static Iterable<Class<?>[]> data() {
- final String valueclassesStr = nativekvtestconf
- .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
- System.out.println(valueclassesStr);
- valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete
- // " "
- final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>();
- for (int i = 0; i < valueclassNames.length; i++) {
- try {
- if (valueclassNames[i].equals("")) {
- continue;
- }
- tmpvalueclasses.add(Class.forName(valueclassNames[i]));
- } catch (final ClassNotFoundException e) {
- e.printStackTrace();
- }
- }
- valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]);
- final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
- System.out.println(keyclassesStr);
- keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete
- // " "
- final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>();
- for (int i = 0; i < keyclassNames.length; i++) {
- try {
- if (keyclassNames[i].equals("")) {
- continue;
- }
- tmpkeyclasses.add(Class.forName(keyclassNames[i]));
- } catch (final ClassNotFoundException e) {
- e.printStackTrace();
- }
- }
- keyclasses = tmpkeyclasses.toArray(new Class[tmpkeyclasses.size()]);
- final Class<?>[][] kvgroup = new Class<?>[keyclassNames.length * valueclassNames.length][2];
- for (int i = 0; i < keyclassNames.length; i++) {
- final int tmpindex = i * valueclassNames.length;
- for (int j = 0; j < valueclassNames.length; j++) {
- kvgroup[tmpindex + j][0] = keyclasses[i];
- kvgroup[tmpindex + j][1] = valueclasses[j];
- }
- }
- return Arrays.asList(kvgroup);
- }
-
- private final Class<?> keyclass;
- private final Class<?> valueclass;
-
- public KVTest(Class<?> keyclass, Class<?> valueclass) {
- this.keyclass = keyclass;
- this.valueclass = valueclass;
-
- }
-
- @Test
- public void testKVCompability() {
-
- try {
- final String nativeoutput = this.runNativeTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final String normaloutput = this.runNormalTest(
- "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
- final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
- final String input = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- if(compareRet){
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(nativeoutput), true);
- fs.delete(new Path(normaloutput), true);
- fs.delete(new Path(input), true);
- fs.close();
- }
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- } catch (final IOException e) {
- assertEquals("test run exception:", null, e);
- } catch (final Exception e) {
- assertEquals("test run exception:", null, e);
- }
- }
-
- @Before
- public void startUp() {
-
- }
-
- private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
- final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(nativekvtestconf);
- fs.delete(new Path(outputpath));
- fs.close();
- nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- try {
- final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "native testcase run time error.";
- }
- return outputpath;
- }
-
- private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
- final String inputpath = hadoopkvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
- + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = hadoopkvtestconf
- .get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR)
- + "/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(hadoopkvtestconf);
- fs.delete(new Path(outputpath));
- fs.close();
- hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
- try {
- final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
- return outputpath;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
deleted file mode 100644
index 900b058..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Test;
-
-public class LargeKVTest {
-
- @Test
- public void testKeySize() {
- runKVSizeTests(Text.class, IntWritable.class);
- }
-
- @Test
- public void testValueSize() {
- runKVSizeTests(IntWritable.class, Text.class);
- }
-
- private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- static {
- nativeConf.addResource(TestConstants.KVTEST_CONF_PATH);
- nativeConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
- normalConf.addResource(TestConstants.KVTEST_CONF_PATH);
- normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
- }
-
- public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
- if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
- return;
- }
- final int deafult_KVSize_Maximum = 1 << 22; // 4M
- final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
- deafult_KVSize_Maximum);
- try {
-
- for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
- int min = i / 4;
- int max = i;
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
- normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
-
- System.out.println("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
- + keyClass.getName() + ", valueClass: " + valueClass.getName());
-
- final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
- valueClass, nativeConf);
- final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
- valueClass, normalConf);
- final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
- final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
- + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
- + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
- assertEquals(reason, true, compareRet);
- }
- } catch (final Exception e) {
- // TODO: handle exception
- // assertEquals("test run exception:", null, e);
- e.printStackTrace();
- }
- }
-
- private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws Exception {
- final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(outputpath), true);
- fs.close();
- try {
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
- return outputpath;
- }
-
- private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
- throws IOException {
- final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
- + "/" + valueclass.getName();
- final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR) + "/LargeKV/"
- + keyclass.getName() + "/" + valueclass.getName();
- // if output file exists ,then delete it
- final FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(outputpath), true);
- fs.close();
- try {
- final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
- keyJob.runJob();
- } catch (final Exception e) {
- return "normal testcase run time error.";
- }
- return outputpath;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
deleted file mode 100644
index a194697..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.VIntWritable;
-import org.apache.hadoop.io.VLongWritable;
-import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-
-
-public class TestInputFile {
-
- public static class KVSizeScope {
- private static final int DefaultMinNum = 1;
- private static final int DefaultMaxNum = 64;
-
- public int minBytesNum;
- public int maxBytesNum;
-
- public KVSizeScope() {
- this.minBytesNum = DefaultMinNum;
- this.maxBytesNum = DefaultMaxNum;
- }
-
- public KVSizeScope(int min, int max) {
- this.minBytesNum = min;
- this.maxBytesNum = max;
- }
- }
-
- private static HashMap<String, KVSizeScope> map = new HashMap<String, KVSizeScope>();
-
- private byte[] databuf = null;
- private final String keyClsName, valueClsName;
- private int filesize = 0;
- private int keyMaxBytesNum, keyMinBytesNum;
- private int valueMaxBytesNum, valueMinBytesNum;
- private SequenceFile.Writer writer = null;
- Random r = new Random();
- public static final int DATABUFSIZE = 1 << 22; // 4M
-
- private enum State {
- KEY, VALUE
- };
-
- static {
- map.put(BooleanWritable.class.getName(), new KVSizeScope(1, 1));
- map.put(DoubleWritable.class.getName(), new KVSizeScope(8, 8));
- map.put(FloatWritable.class.getName(), new KVSizeScope(4, 4));
- map.put(VLongWritable.class.getName(), new KVSizeScope(8, 8));
- map.put(ByteWritable.class.getName(), new KVSizeScope(1, 1));
- map.put(LongWritable.class.getName(), new KVSizeScope(8, 8));
- map.put(VIntWritable.class.getName(), new KVSizeScope(4, 4));
- map.put(IntWritable.class.getName(), new KVSizeScope(4, 4));
- }
-
- public TestInputFile(int filesize, String keytype, String valuetype, Configuration conf) throws Exception {
- this.filesize = filesize;
- this.databuf = new byte[DATABUFSIZE];
- this.keyClsName = keytype;
- this.valueClsName = valuetype;
- final int defaultMinBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MIN, 1);
- final int defaultMaxBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX, 64);
-
- if (map.get(keytype) != null) {
- keyMinBytesNum = map.get(keytype).minBytesNum;
- keyMaxBytesNum = map.get(keytype).maxBytesNum;
- } else {
- keyMinBytesNum = defaultMinBytes;
- keyMaxBytesNum = defaultMaxBytes;
- }
-
- if (map.get(valuetype) != null) {
- valueMinBytesNum = map.get(valuetype).minBytesNum;
- valueMaxBytesNum = map.get(valuetype).maxBytesNum;
- } else {
- valueMinBytesNum = defaultMinBytes;
- valueMaxBytesNum = defaultMaxBytes;
- }
- }
-
- public void createSequenceTestFile(String filepath) throws Exception {
- int FULL_BYTE_SPACE = 256;
- createSequenceTestFile(filepath, FULL_BYTE_SPACE);
- }
-
- public void createSequenceTestFile(String filepath, int base) throws Exception {
- createSequenceTestFile(filepath, base, (byte)0);
- }
-
- public void createSequenceTestFile(String filepath, int base, byte start) throws Exception {
- System.out.println("create file " + filepath);
- System.out.println(keyClsName + " " + valueClsName);
- Class<?> tmpkeycls, tmpvaluecls;
- try {
- tmpkeycls = Class.forName(keyClsName);
- } catch (final ClassNotFoundException e) {
- throw new Exception("key class not found: ", e);
- }
- try {
- tmpvaluecls = Class.forName(valueClsName);
- } catch (final ClassNotFoundException e) {
- throw new Exception("key class not found: ", e);
- }
- try {
- final Path outputfilepath = new Path(filepath);
- final ScenarioConfiguration conf= new ScenarioConfiguration();
- final FileSystem hdfs = outputfilepath.getFileSystem(conf);
- writer = new SequenceFile.Writer(hdfs, conf, outputfilepath, tmpkeycls, tmpvaluecls);
- } catch (final Exception e) {
- e.printStackTrace();
- }
-
- int tmpfilesize = this.filesize;
- while (tmpfilesize > DATABUFSIZE) {
- nextRandomBytes(databuf, base, start);
- final int size = flushBuf(DATABUFSIZE);
- tmpfilesize -= size;
- }
- nextRandomBytes(databuf, base, start);
- flushBuf(tmpfilesize);
-
- if (writer != null) {
- IOUtils.closeStream(writer);
- } else {
- throw new Exception("no writer to create sequenceTestFile!");
- }
- }
-
- private void nextRandomBytes(byte[] buf, int base) {
- nextRandomBytes(buf, base, (byte)0);
- }
-
- private void nextRandomBytes(byte[] buf, int base, byte start) {
- r.nextBytes(buf);
- for (int i = 0; i < buf.length; i++) {
- buf[i] = (byte) ((buf[i] & 0xFF) % base + start);
- }
- }
-
- private int flushBuf(int buflen) throws Exception {
- final Random r = new Random();
- int keybytesnum = 0;
- int valuebytesnum = 0;
- int offset = 0;
-
- while (offset < buflen) {
- final int remains = buflen - offset;
- keybytesnum = keyMaxBytesNum;
- if (keyMaxBytesNum != keyMinBytesNum) {
- keybytesnum = keyMinBytesNum + r.nextInt(keyMaxBytesNum - keyMinBytesNum);
- }
-
- valuebytesnum = valueMaxBytesNum;
- if (valueMaxBytesNum != valueMinBytesNum) {
- valuebytesnum = valueMinBytesNum + r.nextInt(valueMaxBytesNum - valueMinBytesNum);
- }
-
- if (keybytesnum + valuebytesnum > remains) {
- break;
- }
-
- final byte[] key = new byte[keybytesnum];
- final byte[] value = new byte[valuebytesnum];
-
- System.arraycopy(databuf, offset, key, 0, keybytesnum);
- offset += keybytesnum;
-
- System.arraycopy(databuf, offset, value, 0, valuebytesnum);
- offset += valuebytesnum;
-
- try {
- writer.append(BytesFactory.newObject(key, this.keyClsName), BytesFactory.newObject(value, this.valueClsName));
- } catch (final IOException e) {
- e.printStackTrace();
- throw new Exception("sequence file create failed", e);
- }
- }
- return offset;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
deleted file mode 100644
index b7b03e7..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.nonsorttest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class NonSortTest {
-
- @Test
- public void nonSortTest() throws Exception {
- Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.NONSORT_TEST_CONF);
- nativeConf.set(TestConstants.NATIVETASK_MAP_OUTPUT_SORT, "false");
- String inputpath = nativeConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
- String outputpath = nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT);
- final Job nativeNonSort = getJob(nativeConf, "NativeNonSort", inputpath, outputpath);
- nativeNonSort.waitForCompletion(true);
-
- Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
- normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
- inputpath = normalConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
- outputpath = normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT);
- final Job hadoopWithSort = getJob(normalConf, "NormalJob", inputpath, outputpath);
- hadoopWithSort.waitForCompletion(true);
-
- final boolean compareRet = ResultVerifier.verify(nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT),
- normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT));
- assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
- }
-
- @Before
- public void startUp() throws Exception {
- final ScenarioConfiguration configuration = new ScenarioConfiguration();
- configuration.addNonSortTestConf();
- final FileSystem fs = FileSystem.get(configuration);
- final Path path = new Path(configuration.get(TestConstants.NONSORT_TEST_INPUTDIR));
- if (!fs.exists(path)) {
- new TestInputFile(configuration.getInt("nativetask.nonsorttest.filesize", 10000000), Text.class.getName(),
- Text.class.getName(), configuration).createSequenceTestFile(path.toString());
- }
- fs.close();
- }
-
- private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
- final FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputpath))) {
- fs.delete(new Path(outputpath), true);
- }
- fs.close();
- final Job job = new Job(conf, jobName);
- job.setJarByClass(NonSortTestMR.class);
- job.setMapperClass(NonSortTestMR.Map.class);
- job.setReducerClass(NonSortTestMR.KeyHashSumReduce.class);
- job.setOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setOutputValueClass(LongWritable.class);
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setOutputFormatClass(TextOutputFormat.class);
- FileInputFormat.addInputPath(job, new Path(inputpath));
- FileOutputFormat.setOutputPath(job, new Path(outputpath));
- return job;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
deleted file mode 100644
index 4ca2449..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.nonsorttest;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-
-public class NonSortTestMR {
-
- public static class Map extends Mapper<Object, Text, Text, IntWritable> {
- private final static IntWritable one = new IntWritable(1);
- private final Text word = new Text();
-
- @Override
- public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
- final String line = value.toString();
- final StringTokenizer tokenizer = new StringTokenizer(line);
- while (tokenizer.hasMoreTokens()) {
- word.set(tokenizer.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class KeyHashSumReduce extends Reducer<Text, IntWritable, Text, LongWritable> {
- long sum = 0;
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(os);
-
- @Override
- public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
- InterruptedException {
- for (final IntWritable val : values) {
- os.reset();
- key.write(dos);
- final int hash = Arrays.hashCode(os.toByteArray());
- sum += hash;
- }
- }
-
- @Override
- public void cleanup(Context context) throws IOException, InterruptedException {
- context.write(new Text("NonSortTest"), new LongWritable(sum));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
deleted file mode 100644
index 5185371..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.util.Random;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.io.VIntWritable;
-import org.apache.hadoop.io.VLongWritable;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
-
-public class BytesFactory {
- public static Random r = new Random();
-
- public static Object newObject(byte[] seed, String className) {
- r.setSeed(seed.hashCode());
- if (className.equals(IntWritable.class.getName())) {
- return new IntWritable(Ints.fromByteArray(seed));
- } else if (className.equals(FloatWritable.class.getName())) {
- return new FloatWritable(r.nextFloat());
- } else if (className.equals(DoubleWritable.class.getName())) {
- return new DoubleWritable(r.nextDouble());
- } else if (className.equals(LongWritable.class.getName())) {
- return new LongWritable(Longs.fromByteArray(seed));
- } else if (className.equals(VIntWritable.class.getName())) {
- return new VIntWritable(Ints.fromByteArray(seed));
- } else if (className.equals(VLongWritable.class.getName())) {
- return new VLongWritable(Longs.fromByteArray(seed));
- } else if (className.equals(BooleanWritable.class.getName())) {
- return new BooleanWritable(seed[0] % 2 == 1 ? true : false);
- } else if (className.equals(Text.class.getName())) {
- return new Text(BytesUtil.toStringBinary(seed));
- } else if (className.equals(ByteWritable.class.getName())) {
- return new ByteWritable(seed.length > 0 ? seed[0] : 0);
- } else if (className.equals(BytesWritable.class.getName())) {
- return new BytesWritable(seed);
- } else if (className.equals(UTF8.class.getName())) {
- return new UTF8(BytesUtil.toStringBinary(seed));
- } else if (className.equals(MockValueClass.class.getName())) {
- return new MockValueClass(seed);
- } else {
- return null;
- }
- }
-
-
- public static <VTYPE> byte[] fromBytes(byte[] bytes) throws Exception {
- throw new Exception("Not supported");
- }
-
- public static <VTYPE> byte[] toBytes(VTYPE obj) {
- final String className = obj.getClass().getName();
- if (className.equals(IntWritable.class.getName())) {
- return Ints.toByteArray(((IntWritable) obj).get());
- } else if (className.equals(FloatWritable.class.getName())) {
- return BytesUtil.toBytes(((FloatWritable) obj).get());
- } else if (className.equals(DoubleWritable.class.getName())) {
- return BytesUtil.toBytes(((DoubleWritable) obj).get());
- } else if (className.equals(LongWritable.class.getName())) {
- return Longs.toByteArray(((LongWritable) obj).get());
- } else if (className.equals(VIntWritable.class.getName())) {
- return Ints.toByteArray(((VIntWritable) obj).get());
- } else if (className.equals(VLongWritable.class.getName())) {
- return Longs.toByteArray(((VLongWritable) obj).get());
- } else if (className.equals(BooleanWritable.class.getName())) {
- return BytesUtil.toBytes(((BooleanWritable) obj).get());
- } else if (className.equals(Text.class.getName())) {
- return ((Text)obj).copyBytes();
- } else if (className.equals(ByteWritable.class.getName())) {
- return Ints.toByteArray((int) ((ByteWritable) obj).get());
- } else if (className.equals(BytesWritable.class.getName())) {
- // TODO: copyBytes instead?
- return ((BytesWritable) obj).getBytes();
- } else {
- return new byte[0];
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
deleted file mode 100644
index a0f7d64..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
-
-public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
- private boolean nativetaskloaded = false;
-
- @Override
- public void init(Context context)
- throws IOException, ClassNotFoundException {
- try {
- super.init(context);
- nativetaskloaded = true;
- } catch (final Exception e) {
- nativetaskloaded = false;
- System.err.println("load nativetask lib failed, Native-Task Delegation is disabled");
- e.printStackTrace();
- }
- }
-
- @Override
- public void collect(K key, V value, int partition) throws IOException, InterruptedException {
- if (this.nativetaskloaded) {
- super.collect(key, value, partition);
- } else {
- // nothing to do.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
deleted file mode 100644
index b27f00f..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
-public class MockValueClass implements Writable {
- private final static int DEFAULT_ARRAY_LENGTH = 16;
- private int a = 0;
- private byte[] array;
- private final LongWritable longWritable;
- private final Text txt;
- private final Random rand = new Random();
-
- public MockValueClass() {
- a = rand.nextInt();
- array = new byte[DEFAULT_ARRAY_LENGTH];
- rand.nextBytes(array);
- longWritable = new LongWritable(rand.nextLong());
- txt = new Text(BytesUtil.toStringBinary(array));
- }
-
- public MockValueClass(byte[] seed) {
- a = seed.length;
- array = new byte[seed.length];
- System.arraycopy(seed, 0, array, 0, seed.length);
- longWritable = new LongWritable(a);
- txt = new Text(BytesUtil.toStringBinary(array));
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(a);
- out.writeInt(array.length);
- out.write(array);
- longWritable.write(out);
- txt.write(out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- a = in.readInt();
- final int length = in.readInt();
- array = new byte[length];
- in.readFully(array);
- longWritable.readFields(in);
- txt.readFields(in);
- }
-}