You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2014/07/24 08:17:34 UTC
svn commit: r1613004 [2/2] - in
/hadoop/common/branches/MR-2841/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/
hadoop-mapreduce-client/hadoop-mapre...
Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java?rev=1613004&view=auto
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java (added)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java Thu Jul 24 06:17:33 2014
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
+
+public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
+ private boolean nativetaskloaded = false;
+
+ @Override
+ public void init(Context context)
+ throws IOException, ClassNotFoundException {
+ try {
+ super.init(context);
+ nativetaskloaded = true;
+ } catch (final Exception e) {
+ nativetaskloaded = false;
+ System.err.println("load nativetask lib failed, Native-Task Delegation is disabled");
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void collect(K key, V value, int partition) throws IOException, InterruptedException {
+ if (this.nativetaskloaded) {
+ super.collect(key, value, partition);
+ } else {
+ // nothing to do.
+ }
+ }
+}
Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java?rev=1613004&view=auto
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java (added)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java Thu Jul 24 06:17:33 2014
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
+
+public class MockValueClass implements Writable {
+ private final static int DEFAULT_ARRAY_LENGTH = 16;
+ private int a = 0;
+ private byte[] array;
+ private final LongWritable longWritable;
+ private final Text txt;
+ private final Random rand = new Random();
+
+ public MockValueClass() {
+ a = rand.nextInt();
+ array = new byte[DEFAULT_ARRAY_LENGTH];
+ rand.nextBytes(array);
+ longWritable = new LongWritable(rand.nextLong());
+ txt = new Text(BytesUtil.toStringBinary(array));
+ }
+
+ public MockValueClass(byte[] seed) {
+ a = seed.length;
+ array = new byte[seed.length];
+ System.arraycopy(seed, 0, array, 0, seed.length);
+ longWritable = new LongWritable(a);
+ txt = new Text(BytesUtil.toStringBinary(array));
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(a);
+ out.writeInt(array.length);
+ out.write(array);
+ longWritable.write(out);
+ txt.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ a = in.readInt();
+ final int length = in.readInt();
+ array = new byte[length];
+ in.readFully(array);
+ longWritable.readFields(in);
+ txt.readFields(in);
+ }
+}
Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java?rev=1613004&view=auto
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java (added)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java Thu Jul 24 06:17:33 2014
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.IOException;
+import java.util.zip.CRC32;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+
+public class ResultVerifier {
+ /**
+ * verify the result
+ *
+ * @param sample
+ * :nativetask output
+ * @param source
+ * :yuanwenjian
+ * @throws Exception
+ */
+ public static boolean verify(String sample, String source) throws Exception {
+ FSDataInputStream sourcein = null;
+ FSDataInputStream samplein = null;
+
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+ final Path hdfssource = new Path(source);
+ final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource));
+
+ final Path hdfssample = new Path(sample);
+ final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample));
+ if (sourcepaths == null) {
+ throw new Exception("source file can not be found");
+ }
+ if (samplepaths == null) {
+ throw new Exception("sample file can not be found");
+ }
+ if (sourcepaths.length != samplepaths.length) {
+ return false;
+ }
+ for (int i = 0; i < sourcepaths.length; i++) {
+ final Path sourcepath = sourcepaths[i];
+ // op result file start with "part-r" like part-r-00000
+
+ if (!sourcepath.getName().startsWith("part-r")) {
+ continue;
+ }
+ Path samplepath = null;
+ for (int j = 0; j < samplepaths.length; j++) {
+ if (samplepaths[i].getName().equals(sourcepath.getName())) {
+ samplepath = samplepaths[i];
+ break;
+ }
+ }
+ if (samplepath == null) {
+ throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" + sourcepath.getName()
+ + " , as sourcepaths has such file");
+ }
+
+ // compare
+ try {
+ if (fs.exists(sourcepath) && fs.exists(samplepath)) {
+ sourcein = fs.open(sourcepath);
+ samplein = fs.open(samplepath);
+ } else {
+ System.err.println("result file not found:" + sourcepath + " or " + samplepath);
+ return false;
+ }
+
+ CRC32 sourcecrc, samplecrc;
+ samplecrc = new CRC32();
+ sourcecrc = new CRC32();
+ final byte[] bufin = new byte[1 << 16];
+ int readnum = 0;
+ int totalRead = 0;
+ while (samplein.available() > 0) {
+ readnum = samplein.read(bufin);
+ totalRead += readnum;
+ samplecrc.update(bufin, 0, readnum);
+ }
+
+ if (0 == totalRead) {
+ throw new Exception("source " + sample + " is empty file");
+ }
+
+ totalRead = 0;
+ while (sourcein.available() > 0) {
+ readnum = sourcein.read(bufin);
+ totalRead += readnum;
+ sourcecrc.update(bufin, 0, readnum);
+ }
+ if (0 == totalRead) {
+ throw new Exception("source " + sample + " is empty file");
+ }
+
+ if (samplecrc.getValue() == sourcecrc.getValue()) {
+ ;
+ } else {
+ return false;
+ }
+ } catch (final IOException e) {
+ throw new Exception("verify exception :", e);
+ } finally {
+
+ try {
+ if (samplein != null) {
+ samplein.close();
+ }
+ if (sourcein != null) {
+ sourcein.close();
+ }
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+ }
+ return true;
+ }
+
+ public static void main(String[] args) {
+ }
+}
Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java?rev=1613004&view=auto
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java (added)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java Thu Jul 24 06:17:33 2014
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class ScenarioConfiguration extends Configuration {
+ public ScenarioConfiguration() {
+ super();
+ this.addResource(TestConstants.COMMON_CONF_PATH);
+ }
+
+ public void addcombinerConf() {
+ this.addResource(TestConstants.COMBINER_CONF_PATH);
+ }
+
+ public void addKVTestConf() {
+ this.addResource(TestConstants.KVTEST_CONF_PATH);
+ }
+
+ public void addNonSortTestConf() {
+ this.addResource(TestConstants.NONSORT_TEST_CONF);
+ }
+
+ public void addNativeConf() {
+ this.set(TestConstants.NATIVETASK_COLLECTOR_DELEGATOR,
+ TestConstants.NATIVETASK_COLLECTOR_DELEGATOR_CLASS);
+ }
+
+ public static Configuration getNormalConfiguration() {
+ Configuration normalConf = new Configuration();
+ normalConf.addResource("common_conf.xml");
+ normalConf.addResource("normal_conf.xml");
+ return normalConf;
+ }
+
+ public static Configuration getNativeConfiguration() {
+ Configuration nativeConf = new Configuration();
+ nativeConf.addResource("common_conf.xml");
+ nativeConf.addResource("native_conf.xml");
+ return nativeConf;
+ }
+}
Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java?rev=1613004&view=auto
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java (added)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java Thu Jul 24 06:17:33 2014
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+public class TestConstants {
+ // conf path
+ public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml";
+ public static final String KVTEST_CONF_PATH = "kvtest-conf.xml";
+ public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml";
+
+ public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min";
+ public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max";
+
+ public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir";
+ public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir";
+ public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir";
+ public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile";
+ public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum";
+ public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses";
+ public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses";
+ public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
+ public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
+
+ public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml";
+ public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml";
+ public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml";
+ public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml";
+ public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml";
+ public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize";
+
+ public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath";
+ public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input";
+ public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir";
+ public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native";
+ public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir";
+ public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal";
+ public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir";
+ public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir";
+ public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize";
+ public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize";
+
+ public static final String COMMON_CONF_PATH = "common_conf.xml";
+
+ public static final String FILESIZE_KEY = "kvtest.file.size";
+ public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
+
+ public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance";
+ public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath";
+ public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir";
+ public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir";
+
+}