You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by go...@apache.org on 2015/01/16 23:23:07 UTC
tez git commit: TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH
for broadcast edges. (gopalv)
Repository: tez
Updated Branches:
refs/heads/master 4974fb235 -> d676ef2ff
TEZ-1949: Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges. (gopalv)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d676ef2f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d676ef2f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d676ef2f
Branch: refs/heads/master
Commit: d676ef2ffb06e3a4debbd994a8880840e77180b9
Parents: 4974fb2
Author: Gopal V <go...@apache.org>
Authored: Fri Jan 16 14:19:15 2015 -0800
Committer: Gopal V <go...@apache.org>
Committed: Fri Jan 16 14:19:15 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../runtime/library/input/UnorderedKVInput.java | 1 +
.../conf/TestUnorderedKVInputConfig.java | 54 ++++++++++++++++++++
3 files changed, 56 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index eec52dc..1b7f19c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -35,6 +35,7 @@ ALL CHANGES:
TEZ-1889. Fix test-patch to provide correct findbugs report.
TEZ-1313. Setup pre-commit build to test submitted patches.
TEZ-1856. Remove LocalOnFileSortedOutput, LocalMergedInput, LocalTaskOutputFiles.
+ TEZ-1949. Whitelist TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH for broadcast edges.
Release 0.6.0: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
index 90e8f0d..cdc0ac0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
@@ -251,6 +251,7 @@ public class UnorderedKVInput extends AbstractLogicalInput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
http://git-wip-us.apache.org/repos/asf/tez/blob/d676ef2f/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
index 5e6e8e9..8c7dea4 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.junit.Test;
@@ -128,4 +129,57 @@ public class TestUnorderedKVInputConfig {
assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, ""));
assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, ""));
}
+
+ private final void verifySharedFetchConfigs(UserPayload payload) {
+ UnorderedKVInputConfig rebuilt = new UnorderedKVInputConfig();
+ rebuilt.fromUserPayload(payload);
+
+ Configuration conf = rebuilt.conf;
+
+ assertEquals(
+ !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT,
+ conf.getBoolean(
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT));
+
+ assertEquals(
+ !TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT,
+ conf.getBoolean(
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT));
+ }
+
+ @Test
+ public void testSharedFetchConfigs() {
+ UnorderedKVInputConfig.Builder builder = UnorderedKVInputConfig.newBuilder(
+ "KEY", "VALUE");
+
+ UnorderedKVInputConfig configuration = builder
+ .setAdditionalConfiguration(
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+ String
+ .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT))
+ .setAdditionalConfiguration(
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+ String
+ .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT))
+ .build();
+
+ verifySharedFetchConfigs(configuration.toUserPayload());
+ /* test from configuration */
+
+ Configuration conf = new Configuration();
+ conf.set(
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
+ String
+ .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT));
+ conf.set(
+ TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH,
+ String
+ .valueOf(!TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH_DEFAULT));
+ configuration = UnorderedKVInputConfig.newBuilder("KEY", "VALUE")
+ .setFromConfiguration(conf).build();
+
+ verifySharedFetchConfigs(configuration.toUserPayload());
+ }
}