You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/12/16 22:54:32 UTC
tez git commit: TEZ-1861. Fix failing test: TestOnFileSortedOutput.
(sseth) (cherry picked from commit e8d7dc9dc1463d5bb0e36144b0e9d0f9bab599ac)
Repository: tez
Updated Branches:
refs/heads/branch-0.5 8ddfd18c9 -> 880df51b0
TEZ-1861. Fix failing test: TestOnFileSortedOutput. (sseth)
(cherry picked from commit e8d7dc9dc1463d5bb0e36144b0e9d0f9bab599ac)
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/880df51b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/880df51b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/880df51b
Branch: refs/heads/branch-0.5
Commit: 880df51b016bf94927f8213078e48a6847efaa49
Parents: 8ddfd18
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Dec 16 13:53:09 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Dec 16 13:54:22 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 +-
.../sort/impl/dflt/TestDefaultSorter.java | 122 +++++++++++++++++++
.../library/output/TestOnFileSortedOutput.java | 27 ----
3 files changed, 124 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/880df51b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 609df83..760fb60 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,8 +4,9 @@ Apache Tez Change Log
Release 0.5.4: Unreleased
ALL CHANGES:
- TEZ-1800. Integer overflow in ExternalSorter.getInitialMemoryRequirement()
+ TEZ-1861. Fix failing test: TestOnFileSortedOutput.
TEZ-1836. Provide better error messages when tez.runtime.io.sort.mb, spill percentage is incorrectly configured.
+ TEZ-1800. Integer overflow in ExternalSorter.getInitialMemoryRequirement()
Release 0.5.3: 2014-12-10
http://git-wip-us.apache.org/repos/asf/tez/blob/880df51b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
new file mode 100644
index 0000000..16dca55
--- /dev/null
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.library.common.sort.impl.dflt;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.tez.common.TezRuntimeFrameworkConfigs;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.runtime.api.MemoryUpdateCallback;
+import org.apache.tez.runtime.api.OutputContext;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
+import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestDefaultSorter {
+
+ private Configuration conf;
+ private Path workingDir;
+ private static final int PORT = 80;
+ private static final String UniqueID = "UUID";
+
+ @Before
+ public void setup() throws IOException {
+ conf = new Configuration();
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 1); // DefaultSorter
+
+ workingDir = new Path(".", this.getClass().getName());
+ String localDirs = workingDir.toString();
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
+ HashPartitioner.class.getName());
+ conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs);
+ }
+
+ @Test(timeout = 5000)
+ public void testSortSpillPercent() throws Exception {
+ OutputContext context = createTezOutputContext();
+
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.0f);
+ try {
+ new DefaultSorter(context, conf, 10, 2048);
+ fail();
+ } catch(IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
+ }
+
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 1.1f);
+ try {
+ new DefaultSorter(context, conf, 10, 2048);
+ fail();
+ } catch(IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
+ }
+ }
+
+ private OutputContext createTezOutputContext() throws IOException {
+ String[] workingDirs = { workingDir.toString() };
+ UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
+ DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
+ serviceProviderMetaData.writeInt(PORT);
+
+ TezCounters counters = new TezCounters();
+
+ OutputContext context = mock(OutputContext.class);
+ doReturn(counters).when(context).getCounters();
+ doReturn(workingDirs).when(context).getWorkDirs();
+ doReturn(payLoad).when(context).getUserPayload();
+ doReturn(5 * 1024 * 1024l).when(context).getTotalMemoryAvailableToTask();
+ doReturn(UniqueID).when(context).getUniqueIdentifier();
+ doReturn("v1").when(context).getDestinationVertexName();
+ doReturn(ByteBuffer.wrap(serviceProviderMetaData.getData())).when(context)
+ .getServiceProviderMetaData
+ (ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
+ doAnswer(new Answer() {
+ @Override public Object answer(InvocationOnMock invocation) throws Throwable {
+ long requestedSize = (Long) invocation.getArguments()[0];
+ MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
+ .getArguments()[1];
+ callback.memoryAssigned(requestedSize);
+ return null;
+ }
+ }).when(context).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
+ return context;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/880df51b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index afd0090..905fdb9 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -184,33 +184,6 @@ public class TestOnFileSortedOutput {
}
@Test
- public void testSortSpillPercent() throws Exception {
- OutputContext context = createTezOutputContext();
- conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.0f);
- UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
- doReturn(payLoad).when(context).getUserPayload();
- sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
- try {
- sortedOutput.initialize();
- sortedOutput.start();
- fail();
- } catch(IllegalArgumentException e) {
- assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
- }
- conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 1.1f);
- payLoad = TezUtils.createUserPayloadFromConf(conf);
- doReturn(payLoad).when(context).getUserPayload();
- sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
- try {
- sortedOutput.initialize();
- sortedOutput.start();
- fail();
- } catch(IllegalArgumentException e) {
- assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
- }
- }
-
- @Test
public void baseTest() throws Exception {
startSortedOutput(partitions);