You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by rb...@apache.org on 2015/01/08 06:28:44 UTC
tez git commit: TEZ-1912. Merge exceptions are thrown when enabling
tez.runtime.shuffle.memory-to-memory.enable &&
tez.runtime.shuffle.memory-to-memory.segments (Rajesh Balamohan)
Repository: tez
Updated Branches:
refs/heads/master 60e5837a2 -> f1f87c1c8
TEZ-1912. Merge exceptions are thrown when enabling tez.runtime.shuffle.memory-to-memory.enable && tez.runtime.shuffle.memory-to-memory.segments (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f1f87c1c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f1f87c1c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f1f87c1c
Branch: refs/heads/master
Commit: f1f87c1c81c29e1a1be69dc3261a28cd7151f2b9
Parents: 60e5837
Author: Rajesh Balamohan <rb...@hortonworks.com>
Authored: Thu Jan 8 10:58:30 2015 +0530
Committer: Rajesh Balamohan <rb...@hortonworks.com>
Committed: Thu Jan 8 10:58:30 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../common/shuffle/orderedgrouped/InMemoryWriter.java | 4 ++++
.../tez/runtime/library/common/sort/impl/TestIFile.java | 4 ++--
.../org/apache/tez/runtime/library/testutils/KVDataGen.java | 8 +++++++-
4 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/f1f87c1c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 22cb961..a51bf85 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-1912. Merge exceptions are thrown when enabling tez.runtime.shuffle.memory-to-memory.enable && tez.runtime.shuffle.memory-to-memory.segments.
TEZ-1922. Fix comments: add UNSORTED_OUTPUT to TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS.
TEZ-485. Get rid of TezTaskStatus.
TEZ-1899. Fix findbugs warnings in tez-common module.
http://git-wip-us.apache.org/repos/asf/tez/blob/f1f87c1c/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java
index c9896dd..0fd1166 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java
@@ -49,6 +49,10 @@ public class InMemoryWriter extends Writer {
}
public void close() throws IOException {
+
+ // write V_END_MARKER as needed
+ writeValueMarker(out);
+
// Write EOF_MARKER for key/value length
WritableUtils.writeVInt(out, IFile.EOF_MARKER);
WritableUtils.writeVInt(out, IFile.EOF_MARKER);
http://git-wip-us.apache.org/repos/asf/tez/blob/f1f87c1c/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
index 3bcdfc8..558461a 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java
@@ -246,7 +246,7 @@ public class TestIFile {
InMemoryWriter writer = null;
BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);
- List<KVPair> data = KVDataGen.generateTestData(true, 0);
+ List<KVPair> data = KVDataGen.generateTestData(true, 10);
//No RLE, No RepeatKeys, no compression
writer = new InMemoryWriter(bout);
@@ -601,7 +601,7 @@ public class TestIFile {
} else {
writer.append(key, value);
}
- previousKey.reset(k.getData(), 0, k.getLength());
+ previousKey.reset(key.getData(), 0, key.getLength());
}
writer.close();
http://git-wip-us.apache.org/repos/asf/tez/blob/f1f87c1c/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java
index 8857c22..318cfc5 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java
@@ -43,11 +43,12 @@ public class KVDataGen {
public static List<KVPair> generateTestData(boolean sorted, int repeatCount) {
List<KVPair> data = new LinkedList<KVPair>();
Random rnd = new Random();
+ KVPair kvp = null;
for (int i = 0; i < 5; i++) {
String keyStr = (sorted) ? ("key" + i) : (rnd.nextLong() + "key" + i);
Text key = new Text(keyStr);
IntWritable value = new IntWritable(i + repeatCount);
- KVPair kvp = new KVPair(key, value);
+ kvp = new KVPair(key, value);
data.add(kvp);
if ((repeatCount > 0) && (i % 2 == 0)) { // Repeat this key for random number of times
int count = rnd.nextInt(5);
@@ -59,6 +60,11 @@ public class KVDataGen {
}
}
}
+ //If we need to generated repeated keys, try to add some repeated keys to the end of file also.
+ if (repeatCount > 0 && kvp != null) {
+ data.add(kvp);
+ data.add(kvp);
+ }
return data;
}