You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mnemonic.apache.org by ga...@apache.org on 2017/02/07 18:18:28 UTC
incubator-mnemonic git commit: MNEMONIC-188: Verify all partitions
that generated by hadoop writer in test case
Repository: incubator-mnemonic
Updated Branches:
refs/heads/0.5.0-dev 12fb0f6c7 -> 4d1afeed1
MNEMONIC-188: Verify all partitions that generated by hadoop writer in test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/4d1afeed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/4d1afeed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/4d1afeed
Branch: refs/heads/0.5.0-dev
Commit: 4d1afeed1a4b911f7101a2177330130e4cb013b6
Parents: 12fb0f6
Author: Wang, Gang(Gary) <ga...@intel.com>
Authored: Tue Feb 7 10:08:14 2017 -0800
Committer: Wang, Gang(Gary) <ga...@intel.com>
Committed: Tue Feb 7 10:14:17 2017 -0800
----------------------------------------------------------------------
.../mnemonic/mapreduce/MneMapreduceIOTest.java | 36 +++++++++++++-------
.../org/apache/mnemonic/mapreduce/Person.java | 4 +--
2 files changed, 25 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/4d1afeed/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
index 47b3c75..994a42f 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
@@ -53,7 +53,7 @@ public class MneMapreduceIOTest {
private Random m_rand;
private TaskAttemptID m_taid;
private TaskAttemptContext m_tacontext;
- private long m_reccnt = 5000L;
+ private long m_reccnt = 500000L;
private long m_sumage = 0L;
@BeforeClass
@@ -112,20 +112,30 @@ public class MneMapreduceIOTest {
@Test(enabled = true, dependsOnMethods = { "testWritePersonData" })
public void testReadPersonData() throws Exception {
long sumage = 0L;
- FileSplit split = new FileSplit(
- new Path(m_workdir,
- String.format("part-00001-m-00000%s", MneConfigHelper.FILE_EXTENSION)), 0, 0L, new String[0]);
- InputFormat<NullWritable, Person<Long>> inputFormat = new MneInputFormat<Person<Long>>();
- RecordReader<NullWritable, Person<Long>> reader = inputFormat.createRecordReader(split, m_tacontext);
- Person<Long> person = null;
- for (int i = 0; i < m_reccnt; ++i) {
- AssertJUnit.assertTrue(reader.nextKeyValue());
- person = reader.getCurrentValue();
- AssertJUnit.assertTrue(person.getAge() < 51);
- sumage += person.getAge();
+ long reccnt = 0L;
+ File folder = new File(m_workdir.toString());
+ File[] listfiles = folder.listFiles();
+ for (int idx = 0; idx < listfiles.length; ++idx) {
+ if (listfiles[idx].isFile()
+ && listfiles[idx].getName().endsWith(MneConfigHelper.FILE_EXTENSION)) {
+ System.out.println(String.format("Verifying : %s", listfiles[idx].getName()));
+ FileSplit split = new FileSplit(
+ new Path(m_workdir, listfiles[idx].getName()), 0, 0L, new String[0]);
+ InputFormat<NullWritable, Person<Long>> inputFormat = new MneInputFormat<Person<Long>>();
+ RecordReader<NullWritable, Person<Long>> reader = inputFormat.createRecordReader(
+ split, m_tacontext);
+ Person<Long> person = null;
+ while (reader.nextKeyValue()) {
+ person = reader.getCurrentValue();
+ AssertJUnit.assertTrue(person.getAge() < 51);
+ sumage += person.getAge();
+ ++reccnt;
+ }
+ reader.close();
+ }
}
AssertJUnit.assertEquals(m_sumage, sumage);
- reader.close();
+ AssertJUnit.assertEquals(m_reccnt, reccnt);
System.out.println(String.format("The sum of ages is %d", sumage));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/4d1afeed/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
index 17dbc5c..c5edd49 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
@@ -42,12 +42,12 @@ public abstract class Person<E> implements Durable, Comparable<Person<E>>, Writa
@Override
public void initializeAfterCreate() {
- System.out.println("Initializing After Created");
+ /* System.out.println("Initializing After Created"); */
}
@Override
public void initializeAfterRestore() {
- System.out.println("Initializing After Restored");
+ /* System.out.println("Initializing After Restored"); */
}
@Override