You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2016/10/10 17:56:27 UTC
tez git commit: TEZ-3330. Propagate additional config parameters when
running MR jobs via Tez. (sseth)
Repository: tez
Updated Branches:
refs/heads/master 539b8e710 -> e9fb41864
TEZ-3330. Propagate additional config parameters when running MR jobs via Tez. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e9fb4186
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e9fb4186
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e9fb4186
Branch: refs/heads/master
Commit: e9fb418647dec7253956a1a62fbbcd22175144e4
Parents: 539b8e7
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Oct 10 10:56:16 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Oct 10 10:56:16 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/tez/mapreduce/client/YARNRunner.java | 3 ++-
.../tez/runtime/library/common/ConfigUtils.java | 10 ++++++++++
.../tez/runtime/library/conf/BaseConfigBuilder.java | 2 ++
.../library/conf/OrderedGroupedKVInputConfig.java | 16 ++++++++++++++++
.../conf/OrderedPartitionedKVEdgeConfig.java | 7 +++++++
.../conf/OrderedPartitionedKVOutputConfig.java | 16 ++++++++++++++++
.../runtime/library/conf/UnorderedKVEdgeConfig.java | 7 +++++++
.../library/conf/UnorderedKVInputConfig.java | 15 +++++++++++++++
.../library/conf/UnorderedKVOutputConfig.java | 16 ++++++++++++++++
.../conf/UnorderedPartitionedKVEdgeConfig.java | 8 ++++++++
.../conf/UnorderedPartitionedKVOutputConfig.java | 16 ++++++++++++++++
.../conf/TestOrderedGroupedMergedKVInputConfig.java | 7 ++++++-
.../conf/TestOrderedPartitionedKVEdgeConfig.java | 9 ++++++++-
.../conf/TestOrderedPartitionedKVOutputConfig.java | 7 ++++++-
.../library/conf/TestUnorderedKVEdgeConfig.java | 7 ++++++-
.../library/conf/TestUnorderedKVInputConfig.java | 7 ++++++-
.../library/conf/TestUnorderedKVOutputConfig.java | 7 ++++++-
.../conf/TestUnorderedPartitionedKVEdgeConfig.java | 7 ++++++-
.../TestUnorderedPartitionedKVOutputConfig.java | 7 ++++++-
20 files changed, 167 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index addd2c0..10fbd92 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3330. Propagate additional config parameters when running MR jobs via Tez.
TEZ-3335. DAG client thinks app is still running when app status is null
TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317.
TEZ-3460. Fix precommit release audit warning.
@@ -119,6 +120,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3330. Propagate additional config parameters when running MR jobs via Tez.
TEZ-3335. DAG client thinks app is still running when app status is null
TEZ-3460. Fix precommit release audit warning.
TEZ-3368. NPE in DelayedContainerManager
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index 1bea781..820e2e4 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -515,8 +515,9 @@ public class YARNRunner implements ClientProtocol {
TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS),
stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS),
MRPartitioner.class.getName(), partitionerConf)
+ .setFromConfigurationUnfiltered(stageConfs[i-1])
.configureInput().useLegacyInput().done()
- .setFromConfiguration(stageConfs[i - 1]).build();
+ .build();
Edge edge = Edge.create(vertices[i - 1], vertices[i], edgeConf.createDefaultEdgeProperty());
dag.addEdge(edge);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
index c63f737..6aa797f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
@@ -227,6 +227,16 @@ public class ConfigUtils {
}
}
+ public static void mergeConfs(Configuration destConf, Configuration srcConf) {
+ Preconditions.checkState(destConf != null, "Destination conf cannot be null");
+ Preconditions.checkState(srcConf != null, "Source conf cannot be null");
+ for (Map.Entry<String, String> entry : srcConf) {
+ // Explicit get to have parameter replacement work.
+ String val = srcConf.get(entry.getKey());
+ destConf.set(entry.getKey(), val);
+ }
+ }
+
private static Map<String, String> extractConfigurationMapInternal(
Iterable<Map.Entry<String, String>> iterable, List<Set<String>> validKeySets, List<String> allowedPrefixes) {
Set<String> validKeys = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
index d991e89..162addf 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
@@ -59,4 +59,6 @@ interface BaseConfigBuilder<T> {
*/
@InterfaceAudience.LimitedPrivate({"hive, pig"})
public T setFromConfiguration(Configuration conf);
+
+ public T setFromConfigurationUnfiltered(Configuration conf);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
index 95d8e75..11a8d6f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
@@ -212,6 +212,13 @@ public class OrderedGroupedKVInputConfig {
return this;
}
+ @Override
+ public SpecificBuilder setFromConfigurationUnfiltered(
+ Configuration conf) {
+ builder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
public E done() {
return edgeBuilder;
}
@@ -448,6 +455,15 @@ public class OrderedGroupedKVInputConfig {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters
+ Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+ ConfigUtils.mergeConfs(this.conf, conf);
+ return this;
+ }
+
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
@Nullable Map<String, String> codecConf) {
this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
index e6cc2c2..3504203 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
@@ -284,6 +284,13 @@ public class OrderedPartitionedKVEdgeConfig
return this;
}
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ outputBuilder.setFromConfigurationUnfiltered(conf);
+ inputBuilder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
/**
* Configure the specific output
* @return a builder to configure the output
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
index 0687e79..0f37c66 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
@@ -177,6 +177,13 @@ public class OrderedPartitionedKVOutputConfig {
return this;
}
+ @Override
+ public SpecificBuilder setFromConfigurationUnfiltered(
+ Configuration conf) {
+ builder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
public E done() {
return edgeBuilder;
}
@@ -374,6 +381,15 @@ public class OrderedPartitionedKVOutputConfig {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters
+ Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+ ConfigUtils.mergeConfs(this.conf, conf);
+ return this;
+ }
+
/**
* Set the key comparator class
*
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
index 28a7503..25a4823 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
@@ -219,6 +219,13 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
return this;
}
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ outputBuilder.setFromConfigurationUnfiltered(conf);
+ inputBuilder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
/**
* Set serialization class responsible for providing serializer/deserializer for key/value and
* the corresponding comparator class to be used as key comparator.
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
index f52c672..af7dbf6 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
@@ -139,6 +139,12 @@ public class UnorderedKVInputConfig {
return this;
}
+ @Override
+ public SpecificBuilder setFromConfigurationUnfiltered(Configuration conf) {
+ builder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
public E done() {
return edgeBuilder;
}
@@ -296,6 +302,15 @@ public class UnorderedKVInputConfig {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters
+ Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+ ConfigUtils.mergeConfs(this.conf, conf);
+ return this;
+ }
+
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
@Nullable Map<String, String> codecConf) {
this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
index e3eeedc..a9a15a0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
@@ -86,6 +86,13 @@ public class UnorderedKVOutputConfig {
return this;
}
+ @Override
+ public SpecificBuilder setFromConfigurationUnfiltered(
+ Configuration conf) {
+ builder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
public E done() {
return edgeBuilder;
}
@@ -222,6 +229,15 @@ public class UnorderedKVOutputConfig {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters
+ Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+ ConfigUtils.mergeConfs(this.conf, conf);
+ return this;
+ }
+
/**
* Set serialization class responsible for providing serializer/deserializer for keys.
*
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
index 8ac7d65..52da491 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
@@ -224,6 +224,14 @@ public class UnorderedPartitionedKVEdgeConfig
return this;
}
+ @Override
+ public Builder setFromConfigurationUnfiltered(
+ Configuration conf) {
+ outputBuilder.setFromConfigurationUnfiltered(conf);
+ inputBuilder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
/**
* Set serialization class responsible for providing serializer/deserializer for keys.
*
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
index 1cf0924..3555e1c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
@@ -100,6 +100,13 @@ public class UnorderedPartitionedKVOutputConfig {
return this;
}
+ @Override
+ public SpecificBuilder setFromConfigurationUnfiltered(
+ Configuration conf) {
+ builder.setFromConfigurationUnfiltered(conf);
+ return this;
+ }
+
public E done() {
return edgeBuilder;
}
@@ -266,6 +273,15 @@ public class UnorderedPartitionedKVOutputConfig {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public Builder setFromConfigurationUnfiltered(Configuration conf) {
+ // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters
+ Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+ ConfigUtils.mergeConfs(this.conf, conf);
+ return this;
+ }
+
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
@Nullable Map<String, String> codecConf) {
this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
index 49b5490..70dbe14 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
@@ -75,6 +75,8 @@ public class TestOrderedGroupedMergedKVInputConfig {
.TEZ_RUNTIME_SHUFFLE_MIN_REQUIRED_PROGRESS_FRACTION, 0.6f);
fromConf.setBoolean(TezRuntimeConfiguration
.TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, false);
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
Map<String, String> additionalConf = new HashMap<String, String>();
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3");
@@ -92,7 +94,8 @@ public class TestOrderedGroupedMergedKVInputConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
String.valueOf(false))
.setAdditionalConfiguration(additionalConf)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
OrderedGroupedKVInputConfig configuration = builder.build();
@@ -139,6 +142,8 @@ public class TestOrderedGroupedMergedKVInputConfig {
assertNull(conf.get("test.conf.key.1"));
assertNull(conf.get("test.key.1"));
assertNull(conf.get("test.key.2"));
+
+ assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
}
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
index 9d6ca50..9567770 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
@@ -128,6 +128,8 @@ public class TestOrderedPartitionedKVEdgeConfig {
true);
fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB,
1000);
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
Map<String, String> additionalConfs = new HashMap<String, String>();
additionalConfs.put("test.key.2", "key2");
additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
@@ -145,7 +147,8 @@ public class TestOrderedPartitionedKVEdgeConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333")
.setAdditionalConfiguration(additionalConfs)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
OrderedPartitionedKVEdgeConfig configuration = builder.build();
@@ -175,6 +178,9 @@ public class TestOrderedPartitionedKVEdgeConfig {
assertEquals("io", outputConf.get("io.shouldExist"));
assertEquals("file", outputConf.get("file.shouldExist"));
assertEquals("fs", outputConf.get("fs.shouldExist"));
+
+ assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));
+
ReportPartitionStats partitionStats =
ReportPartitionStats.fromString(outputConf.get(
TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS,
@@ -198,6 +204,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
assertEquals("file", inputConf.get("file.shouldExist"));
assertEquals("fs", inputConf.get("fs.shouldExist"));
+ assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
}
@Test (timeout=2000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
index 891e807..2dca183 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
@@ -73,6 +73,8 @@ public class TestOrderedPartitionedKVOutputConfig {
additionalConf.put("test.key.2", "key2");
additionalConf.put("io.shouldExist", "io");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "TestInternalSorter");
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
OrderedPartitionedKVOutputConfig.Builder builder =
OrderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER", null)
.setKeyComparatorClass("KEY_COMPARATOR", null)
@@ -83,7 +85,8 @@ public class TestOrderedPartitionedKVOutputConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
String.valueOf(false))
.setAdditionalConfiguration(additionalConf)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
OrderedPartitionedKVOutputConfig configuration = builder.build();
@@ -116,6 +119,8 @@ public class TestOrderedPartitionedKVOutputConfig {
assertNull(conf.get("test.conf.key.1"));
assertNull(conf.get("test.key.1"));
assertNull(conf.get("test.key.2"));
+
+ assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
}
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
index 0ef1cc5..3c97020 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
@@ -121,6 +121,8 @@ public class TestUnorderedKVEdgeConfig {
fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false);
fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f);
fromConf.set("io.shouldExist", "io");
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
Map<String, String> additionalConfs = new HashMap<String, String>();
additionalConfs.put("test.key.2", "key2");
additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
@@ -135,7 +137,8 @@ public class TestUnorderedKVEdgeConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333")
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
.setAdditionalConfiguration(additionalConfs)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
UnorderedKVEdgeConfig configuration = builder.build();
@@ -159,6 +162,7 @@ public class TestUnorderedKVEdgeConfig {
assertEquals("file", outputConf.get("file.shouldExist"));
assertEquals("fs", outputConf.get("fs.shouldExist"));
+ assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));
assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true));
assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
@@ -173,6 +177,7 @@ public class TestUnorderedKVEdgeConfig {
assertEquals("file", inputConf.get("file.shouldExist"));
assertEquals("fs", inputConf.get("fs.shouldExist"));
+ assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
}
private void checkHistoryText(String historyText) {
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
index 8c7dea4..bb75442 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
@@ -62,6 +62,8 @@ public class TestUnorderedKVInputConfig {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3");
additionalConf.put("file.shouldExist", "file");
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
UnorderedKVInputConfig.Builder builder =
UnorderedKVInputConfig.newBuilder("KEY", "VALUE")
.setCompression(true, "CustomCodec", null)
@@ -73,7 +75,8 @@ public class TestUnorderedKVInputConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
String.valueOf(false))
.setAdditionalConfiguration(additionalConf)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
UnorderedKVInputConfig configuration = builder.build();
@@ -105,6 +108,8 @@ public class TestUnorderedKVInputConfig {
assertNull(conf.get("test.conf.key.1"));
assertNull(conf.get("test.key.1"));
assertNull(conf.get("test.key.2"));
+
+ assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
}
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
index 10ed199..f99b3ae 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
@@ -63,6 +63,8 @@ public class TestUnorderedKVOutputConfig {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222");
additionalConf.put("file.shouldExist", "file");
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
UnorderedKVOutputConfig.Builder builder =
UnorderedKVOutputConfig.newBuilder("KEY", "VALUE")
.setCompression(true, "CustomCodec", null)
@@ -71,7 +73,8 @@ public class TestUnorderedKVOutputConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
String.valueOf(false))
.setAdditionalConfiguration(additionalConf)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
UnorderedKVOutputConfig configuration = builder.build();
@@ -100,6 +103,8 @@ public class TestUnorderedKVOutputConfig {
assertNull(conf.get("test.conf.key.1"));
assertNull(conf.get("test.key.1"));
assertNull(conf.get("test.key.2"));
+
+ assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
}
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
index fff8efd..684f773 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
@@ -136,6 +136,8 @@ public class TestUnorderedPartitionedKVEdgeConfig {
additionalConfs
.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222");
additionalConfs.put("file.shouldExist", "file");
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig
.newBuilder("KEY", "VALUE", "PARTITIONER")
@@ -144,7 +146,8 @@ public class TestUnorderedPartitionedKVEdgeConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333")
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
.setAdditionalConfiguration(additionalConfs)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
UnorderedPartitionedKVEdgeConfig configuration = builder.build();
@@ -172,6 +175,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
assertEquals("file", outputConf.get("file.shouldExist"));
assertEquals("fs", outputConf.get("fs.shouldExist"));
+ assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));
assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true));
assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
@@ -188,6 +192,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
assertEquals("file", inputConf.get("file.shouldExist"));
assertEquals("fs", inputConf.get("fs.shouldExist"));
+ assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
}
private void checkHistoryText(String historyText) {
http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
index f509acb..a9e955e 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
@@ -73,6 +73,8 @@ public class TestUnorderedPartitionedKVOutputConfig {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222");
additionalConf.put("file.shouldExist", "file");
+ Configuration fromConfUnfiltered = new Configuration(false);
+ fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
UnorderedPartitionedKVOutputConfig.Builder builder =
UnorderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER",
null)
@@ -87,7 +89,8 @@ public class TestUnorderedPartitionedKVOutputConfig {
.setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
String.valueOf(false))
.setAdditionalConfiguration(additionalConf)
- .setFromConfiguration(fromConf);
+ .setFromConfiguration(fromConf)
+ .setFromConfigurationUnfiltered(fromConfUnfiltered);
UnorderedPartitionedKVOutputConfig configuration = builder.build();
@@ -125,6 +128,8 @@ public class TestUnorderedPartitionedKVOutputConfig {
assertNull(conf.get("test.conf.key.1"));
assertNull(conf.get("test.key.1"));
assertNull(conf.get("test.key.2"));
+
+ assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
}
@Test(timeout = 5000)