You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/07/24 19:42:11 UTC
[4/4] git commit: TEZ-1137. Move TezJobConfig to runtime-library and
rename to TezRuntimeConfiguration (bikas)
TEZ-1137. Move TezJobConfig to runtime-library and rename to TezRuntimeConfiguration (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2050f50e
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2050f50e
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2050f50e
Branch: refs/heads/master
Commit: 2050f50e9e7d25c7b48ccc71fb507b4bf8bba23e
Parents: 06bd383
Author: Bikas Saha <bi...@apache.org>
Authored: Thu Jul 24 10:42:02 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu Jul 24 10:42:02 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tez/client/TezClientUtils.java | 3 +-
.../org/apache/tez/common/TezJobConfig.java | 470 -------------------
.../org/apache/tez/common/counters/Limits.java | 18 +-
.../apache/tez/dag/api/TezConfiguration.java | 91 ++++
.../org/apache/tez/common/TestTezJobConfig.java | 54 ---
.../security/authorize/TezAMPolicyProvider.java | 2 +-
.../apache/tez/mapreduce/client/YARNRunner.java | 6 +-
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 68 +--
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 12 +-
.../hadoop/MultiStageMRConfToTezTranslator.java | 8 +-
.../mapreduce/hadoop/TestDeprecatedKeys.java | 108 ++---
.../processor/map/TestMapProcessor.java | 4 +-
.../processor/reduce/TestReduceProcessor.java | 4 +-
.../common/resources/MemoryDistributor.java | 16 +-
.../common/resources/ScalingAllocator.java | 10 +-
.../tez/runtime/metrics/TaskCounterUpdater.java | 4 +-
.../common/resources/TestMemoryDistributor.java | 10 +-
.../tez/runtime/library/api/Partitioner.java | 3 +-
.../library/api/TezRuntimeConfiguration.java | 372 +++++++++++++++
.../tez/runtime/library/common/ConfigUtils.java | 25 +-
.../runtime/library/common/TezRuntimeUtils.java | 10 +-
.../library/common/combine/Combiner.java | 4 +-
.../common/localshuffle/LocalShuffle.java | 16 +-
.../common/shuffle/impl/MergeManager.java | 57 ++-
.../library/common/shuffle/impl/Shuffle.java | 18 +-
.../shuffle/impl/ShuffleClientMetrics.java | 6 +-
.../common/shuffle/impl/ShuffleScheduler.java | 14 +-
.../common/sort/impl/ExternalSorter.java | 20 +-
.../common/sort/impl/PipelinedSorter.java | 28 +-
.../library/common/sort/impl/TezMerger.java | 10 +-
.../common/sort/impl/dflt/DefaultSorter.java | 24 +-
.../BaseUnorderedPartitionedKVWriter.java | 14 +-
.../writers/UnorderedPartitionedKVWriter.java | 12 +-
.../conf/OnFileSortedOutputConfiguration.java | 37 +-
.../OnFileUnorderedKVOutputConfiguration.java | 23 +-
...orderedPartitionedKVOutputConfiguration.java | 29 +-
.../conf/ShuffledMergedInputConfiguration.java | 41 +-
.../ShuffledUnorderedKVInputConfiguration.java | 31 +-
.../library/input/ShuffledMergedInput.java | 75 +--
.../library/input/ShuffledUnorderedKVInput.java | 65 +--
.../library/output/OnFileSortedOutput.java | 56 +--
.../library/output/OnFileUnorderedKVOutput.java | 35 +-
.../OnFileUnorderedPartitionedKVOutput.java | 36 +-
.../WeightedScalingMemoryDistributor.java | 15 +-
.../runtime/library/shuffle/common/Fetcher.java | 6 +-
.../library/shuffle/common/ShuffleUtils.java | 26 +-
.../shuffle/common/impl/ShuffleManager.java | 6 +-
.../impl/SimpleFetchedInputAllocator.java | 20 +-
.../TestWeightedScalingMemoryDistributor.java | 21 +-
.../api/TestTezRuntimeConfiguration.java | 55 +++
.../TestUnorderedPartitionedKVWriter.java | 14 +-
.../TestOnFileSortedOutputConfiguration.java | 52 +-
...estOnFileUnorderedKVOutputConfiguration.java | 34 +-
.../TestOnFileUnorderedPartitionedKVOutput.java | 46 +-
.../TestOrderedPartitionedKVEdgeConfigurer.java | 114 ++---
.../TestShuffledMergedInputConfiguration.java | 48 +-
...stShuffledUnorderedKVInputConfiguration.java | 42 +-
...estUnorderedPartitionedKVEdgeConfigurer.java | 66 +--
...tUnorderedUnpartitionedKVEdgeConfigurer.java | 54 +--
.../output/TestOnFileUnorderedKVOutput.java | 6 +-
.../impl/TestSimpleFetchedInputAllocator.java | 6 +-
.../org/apache/tez/test/TestSecureShuffle.java | 8 +-
63 files changed, 1297 insertions(+), 1293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fdb1964..0a76df8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -26,6 +26,8 @@ INCOMPATIBLE CHANGES
TEZ-1269. TaskScheduler prematurely releases containers
TEZ-857. Split Input/Output interfaces into user/framework components.
TEZ-866. Add a TezMergedInputContext for MergedInputs (bikas)
+ TEZ-1137. Move TezJobConfig to runtime-library and rename to
+ TezRuntimeConfiguration (bikas)
Release 0.4.0-incubating: 2014-04-05
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 166f091..93c5f34 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -75,7 +75,6 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
import org.apache.tez.common.TezCommonUtils;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -200,7 +199,7 @@ public class TezClientUtils {
static void processTezLocalCredentialsFile(Credentials credentials, Configuration conf)
throws IOException {
- String path = conf.get(TezJobConfig.TEZ_CREDENTIALS_PATH);
+ String path = conf.get(TezConfiguration.TEZ_CREDENTIALS_PATH);
if (path == null) {
return;
} else {
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
deleted file mode 100644
index b36e789..0000000
--- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ /dev/null
@@ -1,470 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.common;
-
-
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Meant for user configurable job properties.
- * <p/>
- * Note for developers: Whenever a new key is added to this file, it must also be added to the set of
- * known tezRuntimeKeys.
- */
-
-// TODO EVENTUALLY A description for each property.
-@InterfaceAudience.Public
-@Evolving
-public class TezJobConfig {
-
- private static final String TEZ_RUNTIME_PREFIX = "tez.runtime.";
-
- private static final Set<String> tezRuntimeKeys = new HashSet<String>();
- private static Set<String> umnodifiableTezRuntimeKeySet;
- private static final Set<String> otherKeys = new HashSet<String>();
- private static Set<String> unmodifiableOtherKeySet;
- private static Configuration defaultConf = new Configuration(false);
- private static final Map<String, String> tezRuntimeConfMap = new HashMap<String, String>();
- private static final Map<String, String> otherConfMap = new HashMap<String, String>();
-
- /**
- * Prefixes from Hadoop configuration which are allowed.
- */
- private static final List<String> allowedPrefixes = new ArrayList<String>();
- private static List<String> unmodifiableAllowedPrefixes;
-
-
- /**
- * Configuration key to enable/disable IFile readahead.
- */
- public static final String TEZ_RUNTIME_IFILE_READAHEAD = TEZ_RUNTIME_PREFIX +
- "ifile.readahead";
- public static final boolean TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT = true;
-
- /**
- * Configuration key to set the IFile readahead length in bytes.
- */
- public static final String TEZ_RUNTIME_IFILE_READAHEAD_BYTES = TEZ_RUNTIME_PREFIX +
- "ifile.readahead.bytes";
- public static final int TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT =
- 4 * 1024 * 1024;
-
- public static final int TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT = -1;
-
- /**
- * This is copy of io.file.buffer.size from Hadoop, which is used in several places such
- * as compression codecs, buffer sizes in IFile, while fetching etc.
- * Variable exists so that it can be referenced, instead of using the string name directly.
- */
- public static final String TEZ_RUNTIME_IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
-
-
- public static final String TEZ_RUNTIME_IO_SORT_FACTOR = TEZ_RUNTIME_PREFIX +
- "io.sort.factor";
- public static final int TEZ_RUNTIME_IO_SORT_FACTOR_DEFAULT = 100;
-
-
- public static final String TEZ_RUNTIME_SORT_SPILL_PERCENT = TEZ_RUNTIME_PREFIX +
- "sort.spill.percent";
- public static float TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT = 0.8f;
-
-
- public static final String TEZ_RUNTIME_IO_SORT_MB = TEZ_RUNTIME_PREFIX + "io.sort.mb";
- public static final int TEZ_RUNTIME_IO_SORT_MB_DEFAULT = 100;
-
-
- public static final String TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES = TEZ_RUNTIME_PREFIX +
- "index.cache.memory.limit.bytes";
- public static final int TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT =
- 1024 * 1024;
-
-
- // TODO Use the default value
- public static final String TEZ_RUNTIME_COMBINE_MIN_SPILLS = TEZ_RUNTIME_PREFIX +
- "combine.min.spills";
- public static final int TEZ_RUNTIME_COMBINE_MIN_SPILLS_DEFAULT = 3;
-
-
- public static final String TEZ_RUNTIME_SORT_THREADS = TEZ_RUNTIME_PREFIX +
- "sort.threads";
- public static final int TEZ_RUNTIME_SORT_THREADS_DEFAULT = 1;
-
- /**
- * Size of the buffer to use if not writing directly to disk.
- */
- public static final String TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB = TEZ_RUNTIME_PREFIX +
- "unordered.output.buffer.size-mb";
- public static final int TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB_DEFAULT = 100;
-
- /**
- * Maximum size for individual buffers used in the UnsortedPartitionedOutput.
- * This is only meant to be used by unit tests for now.
- */
- @Private
- public static final String TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES =
- TEZ_RUNTIME_PREFIX +
- "unordered.output.max-per-buffer.size-bytes";
-
- /**
- * Specifies a partitioner class, which is used in Tez Runtime components
- * like OnFileSortedOutput
- */
- public static final String TEZ_RUNTIME_PARTITIONER_CLASS =
- TEZ_RUNTIME_PREFIX + "partitioner.class";
-
- /**
- * Specifies a combiner class (primarily for Shuffle)
- */
- public static final String TEZ_RUNTIME_COMBINER_CLASS = TEZ_RUNTIME_PREFIX + "combiner.class";
-
- public static final String TEZ_RUNTIME_COUNTERS_MAX_KEY = TEZ_RUNTIME_PREFIX + "job.counters.max";
- public static final int TEZ_RUNTIME_COUNTERS_MAX_DEFAULT = 1200;
-
-
- public static final String TEZ_RUNTIME_COUNTER_GROUP_NAME_MAX_KEY =
- TEZ_RUNTIME_PREFIX + "job.counters.group.name.max";
- public static final int TEZ_RUNTIME_COUNTER_GROUP_NAME_MAX_DEFAULT = 128;
-
-
- public static final String TEZ_RUNTIME_COUNTER_NAME_MAX_KEY =
- TEZ_RUNTIME_PREFIX + "job.counters.counter.name.max";
- public static final int TEZ_RUNTIME_COUNTER_NAME_MAX_DEFAULT = 64;
-
-
- public static final String TEZ_RUNTIME_COUNTER_GROUPS_MAX_KEY =
- TEZ_RUNTIME_PREFIX + "job.counters.groups.max";
- public static final int TEZ_RUNTIME_COUNTER_GROUPS_MAX_DEFAULT = 500;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES = TEZ_RUNTIME_PREFIX +
- "shuffle.parallel.copies";
- public static final int TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT = 20;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT = TEZ_RUNTIME_PREFIX +
- "shuffle.fetch.failures.limit";
- public static final int TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT_DEFAULT = 5;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE =
- TEZ_RUNTIME_PREFIX +
- "shuffle.fetch.max.task.output.at.once";
- public final static int TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE_DEFAULT
- = 20;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR = TEZ_RUNTIME_PREFIX +
- "shuffle.notify.readerror";
- public static final boolean TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR_DEFAULT = true;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT = TEZ_RUNTIME_PREFIX +
- "shuffle.connect.timeout";
- public static final int TEZ_RUNTIME_SHUFFLE_STALLED_COPY_TIMEOUT_DEFAULT =
- 3 * 60 * 1000;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED = TEZ_RUNTIME_PREFIX +
- "shuffle.keep-alive.enabled";
- public static final boolean TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED_DEFAULT = false;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS = TEZ_RUNTIME_PREFIX +
- "shuffle.keep-alive.max.connections";
- public static final int TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS_DEFAULT = 20;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT =
- TEZ_RUNTIME_PREFIX + "shuffle.read.timeout";
- public final static int TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT_DEFAULT =
- 3 * 60 * 1000;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE =
- TEZ_RUNTIME_PREFIX + "shuffle.buffersize";
- public final static int TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE_DEFAULT =
- 8 * 1024;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_SSL = TEZ_RUNTIME_PREFIX +
- "shuffle.ssl.enable";
- public static final boolean TEZ_RUNTIME_SHUFFLE_ENABLE_SSL_DEFAULT = false;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT = TEZ_RUNTIME_PREFIX +
- "shuffle.input.buffer.percent";
- public static final float TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT_DEFAULT =
- 0.90f;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT = TEZ_RUNTIME_PREFIX +
- "shuffle.memory.limit.percent";
- public static final float TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT_DEFAULT =
- 0.25f;
-
- // Rename to fraction
- public static final String TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT = TEZ_RUNTIME_PREFIX +
- "shuffle.merge.percent";
- public static final float TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT_DEFAULT = 0.90f;
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS = TEZ_RUNTIME_PREFIX +
- "shuffle.memory-to-memory.segments";
-
-
- public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM = TEZ_RUNTIME_PREFIX +
- "shuffle.memory-to-memory.enable";
- public static final boolean TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM_DEFAULT =
- false;
-
-
- public static final String TEZ_RUNTIME_INPUT_BUFFER_PERCENT = TEZ_RUNTIME_PREFIX +
- "task.input.buffer.percent";
- public static final float TEZ_RUNTIME_INPUT_BUFFER_PERCENT_DEFAULT = 0.0f;
-
-
- public static final String TEZ_RUNTIME_GROUP_COMPARATOR_CLASS = TEZ_RUNTIME_PREFIX +
- "group.comparator.class";
-
- public static final String TEZ_RUNTIME_INTERNAL_SORTER_CLASS = TEZ_RUNTIME_PREFIX +
- "internal.sorter.class";
-
- public static final String TEZ_RUNTIME_KEY_COMPARATOR_CLASS =
- TEZ_RUNTIME_PREFIX + "key.comparator.class";
-
- public static final String TEZ_RUNTIME_KEY_CLASS = TEZ_RUNTIME_PREFIX + "key.class";
-
- public static final String TEZ_RUNTIME_VALUE_CLASS = TEZ_RUNTIME_PREFIX + "value.class";
-
- public static final String TEZ_RUNTIME_COMPRESS = TEZ_RUNTIME_PREFIX + "compress";
-
- public static final String TEZ_RUNTIME_COMPRESS_CODEC = TEZ_RUNTIME_PREFIX + "compress.codec";
-
- // TODO Move this key to MapReduce
- public static final String TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS =
- TEZ_RUNTIME_PREFIX + "key.secondary.comparator.class";
-
- public static final String TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED =
- TEZ_RUNTIME_PREFIX +
- "empty.partitions.info-via-events.enabled";
- public static final boolean TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT = true;
-
- public static final String TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED =
- TEZ_RUNTIME_PREFIX + "broadcast.data-via-events.enabled";
- public static final boolean TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED_DEFAULT = false;
-
- public static final String TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE =
- TEZ_RUNTIME_PREFIX + "broadcast.data-via-events.max-size";
- public static final int TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT = 200 << 10; // 200KB
-
-
- // TODO TEZ-1233 - allow this property to be set per vertex
- // TODO TEZ-1231 - move these properties out since they are not relevant for Inputs / Outputs
-
- /**
- * Defines the ProcessTree implementation which will be used to collect resource utilization.
- */
- public static final String TEZ_RUNTIME_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS =
- TEZ_RUNTIME_PREFIX + "resource.calculator.process-tree.class";
-
- /**
- * Whether to scale down memory requested by each component if the total
- * exceeds the available JVM memory
- */
- @Private
- @Unstable
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED = TEZ_RUNTIME_PREFIX
- + "scale.task.memory.enabled";
- public static final boolean TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED_DEFAULT = true;
-
- /**
- * The allocator to use for initial memory allocation
- */
- @Private
- @Unstable
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_ALLOCATOR_CLASS = TEZ_RUNTIME_PREFIX
- + "scale.task.memory.allocator.class";
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_ALLOCATOR_CLASS_DEFAULT =
- "org.apache.tez.runtime.common.resources.ScalingAllocator";
-
- /**
- * The fraction of the JVM memory which will not be considered for allocation.
- * No defaults, since there are pre-existing defaults based on different scenarios.
- */
- @Private
- @Unstable
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_RESERVE_FRACTION = TEZ_RUNTIME_PREFIX
- + "scale.task.memory.reserve-fraction";
-
- /**
- * Fraction of available memory to reserve per input/output. This amount is
- * removed from the total available pool before allocation and is for factoring in overheads.
- */
- @Private
- @Unstable
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO =
- TEZ_RUNTIME_PREFIX
- + "scale.task.memory.additional.reservation.fraction.per-io";
-
- /**
- * Max cumulative total reservation for additional IOs.
- */
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX =
- TEZ_RUNTIME_PREFIX
- + "scale.task.memory.additional reservation.fraction.max";
- /*
- * Weighted ratios for individual component types in the RuntimeLibrary.
- * e.g. PARTITIONED_UNSORTED_OUTPUT:0,UNSORTED_INPUT:1,SORTED_OUTPUT:2,SORTED_MERGED_INPUT:3,PROCESSOR:1,OTHER:1
- */
- @Private
- @Unstable
- public static final String TEZ_RUNTIME_SCALE_TASK_MEMORY_WEIGHTED_RATIOS = TEZ_RUNTIME_PREFIX
- + "scale.task.memory.ratios";
-
- /**
- * Path to a credentials file located on the local file system with serialized credentials.
- * Note: This property does not follow the standard convention of including tez.runtime in it's
- * name.
- */
- public static final String TEZ_CREDENTIALS_PATH = "tez.credentials.path";
-
-
- @Unstable
- @Private
- public static final String TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS = TEZ_RUNTIME_PREFIX +
- "merge.progress.records";
- public static final long TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT = 10000;
-
- static {
- tezRuntimeKeys.add(TEZ_RUNTIME_IFILE_READAHEAD);
- tezRuntimeKeys.add(TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
- tezRuntimeKeys.add(TEZ_RUNTIME_IO_FILE_BUFFER_SIZE);
- tezRuntimeKeys.add(TEZ_RUNTIME_IO_SORT_FACTOR);
- tezRuntimeKeys.add(TEZ_RUNTIME_SORT_SPILL_PERCENT);
- tezRuntimeKeys.add(TEZ_RUNTIME_IO_SORT_MB);
- tezRuntimeKeys.add(TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
- tezRuntimeKeys.add(TEZ_RUNTIME_COMBINE_MIN_SPILLS);
- tezRuntimeKeys.add(TEZ_RUNTIME_SORT_THREADS);
- tezRuntimeKeys.add(TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB);
- tezRuntimeKeys.add(TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES);
- tezRuntimeKeys.add(TEZ_RUNTIME_PARTITIONER_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_COMBINER_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_COUNTERS_MAX_KEY);
- tezRuntimeKeys.add(TEZ_RUNTIME_COUNTER_GROUP_NAME_MAX_KEY);
- tezRuntimeKeys.add(TEZ_RUNTIME_COUNTER_NAME_MAX_KEY);
- tezRuntimeKeys.add(TEZ_RUNTIME_COUNTER_GROUPS_MAX_KEY);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_ENABLE_SSL);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS);
- tezRuntimeKeys.add(TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM);
- tezRuntimeKeys.add(TEZ_RUNTIME_INPUT_BUFFER_PERCENT);
- tezRuntimeKeys.add(TEZ_RUNTIME_GROUP_COMPARATOR_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_INTERNAL_SORTER_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_KEY_COMPARATOR_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_KEY_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_VALUE_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_COMPRESS);
- tezRuntimeKeys.add(TEZ_RUNTIME_COMPRESS_CODEC);
- tezRuntimeKeys.add(TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
- tezRuntimeKeys.add(TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED);
- tezRuntimeKeys.add(TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE);
- tezRuntimeKeys.add(TEZ_RUNTIME_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED);
- tezRuntimeKeys.add(TEZ_RUNTIME_SCALE_TASK_MEMORY_ALLOCATOR_CLASS);
- tezRuntimeKeys.add(TEZ_RUNTIME_SCALE_TASK_MEMORY_RESERVE_FRACTION);
- tezRuntimeKeys.add(TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO);
- tezRuntimeKeys.add(TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX);
- tezRuntimeKeys.add(TEZ_RUNTIME_SCALE_TASK_MEMORY_WEIGHTED_RATIOS);
- tezRuntimeKeys.add(TEZ_CREDENTIALS_PATH);
- tezRuntimeKeys.add(TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS);
-
- defaultConf.addResource("core-default.xml");
- defaultConf.addResource("core-site.xml");
- defaultConf.addResource("tez-site.xml");
-
- for (Map.Entry<String, String> confEntry : defaultConf) {
- if (tezRuntimeKeys.contains(confEntry.getKey())) {
- tezRuntimeConfMap.put(confEntry.getKey(), confEntry.getValue());
- } else {
- // TODO TEZ-1232 Filter out parameters from TezConfiguration, and Task specific confs
- otherConfMap.put(confEntry.getKey(), confEntry.getValue());
- otherKeys.add(confEntry.getKey());
- }
- }
-
- // Do NOT need all prefixes from the following list. Only specific ones are allowed
- // "hadoop.", "hadoop.security", "io.", "fs.", "ipc.", "net.", "file.", "dfs.", "ha.", "s3.", "nfs3.", "rpc."
- allowedPrefixes.add("io.");
- allowedPrefixes.add("file.");
- allowedPrefixes.add("fs.");
-
- umnodifiableTezRuntimeKeySet = Collections.unmodifiableSet(tezRuntimeKeys);
- unmodifiableOtherKeySet = Collections.unmodifiableSet(otherKeys);
- unmodifiableAllowedPrefixes = Collections.unmodifiableList(allowedPrefixes);
- }
-
- @Private
- public static Set<String> getRuntimeConfigKeySet() {
- return umnodifiableTezRuntimeKeySet;
- }
-
- @Private
- public static Set<String> getRuntimeAdditionalConfigKeySet() {
- return unmodifiableOtherKeySet;
- }
-
- @Private
- public static List<String> getAllowedPrefixes() {
- return allowedPrefixes;
- }
-
- @Private
- public static Map<String, String> getTezRuntimeConfigDefaults() {
- return Collections.unmodifiableMap(tezRuntimeConfMap);
- }
-
- @Private
- public static Map<String, String> getOtherConfigDefaults() {
- return Collections.unmodifiableMap(otherConfMap);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
index 54321a8..4814548 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
@@ -22,7 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.dag.api.TezConfiguration;
@InterfaceAudience.Private
public class Limits {
@@ -44,17 +44,17 @@ public class Limits {
conf = new Configuration();
}
GROUP_NAME_MAX =
- conf.getInt(TezJobConfig.TEZ_RUNTIME_COUNTER_GROUP_NAME_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTER_GROUP_NAME_MAX_DEFAULT);
+ conf.getInt(TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS,
+ TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS_DEFAULT);
COUNTER_NAME_MAX =
- conf.getInt(TezJobConfig.TEZ_RUNTIME_COUNTER_NAME_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTER_NAME_MAX_DEFAULT);
+ conf.getInt(TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS,
+ TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS_DEFAULT);
GROUPS_MAX =
- conf.getInt(TezJobConfig.TEZ_RUNTIME_COUNTER_GROUPS_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTER_GROUPS_MAX_DEFAULT);
+ conf.getInt(TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS,
+ TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS_DEFAULT);
COUNTERS_MAX =
- conf.getInt(TezJobConfig.TEZ_RUNTIME_COUNTERS_MAX_KEY, TezJobConfig.
- TEZ_RUNTIME_COUNTERS_MAX_DEFAULT);
+ conf.getInt(TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS,
+ TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS_DEFAULT);
initialized = true;
LOG.info("Counter limits initialized with parameters: " + " GROUP_NAME_MAX=" + GROUP_NAME_MAX
+ ", MAX_GROUPS=" + GROUPS_MAX + ", COUNTER_NAME_MAX=" + COUNTER_NAME_MAX
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index d3a0c6d..50489d3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -46,6 +46,13 @@ public class TezConfiguration extends Configuration {
/** The staging dir used while submitting DAGs */
public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/tez/staging";
+
+ /**
+ * Path to a credentials file located on the local file system with serialized credentials.
+ * Note: This property does not follow the standard convention of including tez.runtime in it's
+ * name.
+ */
+ public static final String TEZ_CREDENTIALS_PATH = TEZ_PREFIX + "credentials.path";
public static final String TEZ_APPLICATION_MASTER_CLASS =
"org.apache.tez.dag.app.DAGAppMaster";
@@ -115,6 +122,27 @@ public class TezConfiguration extends Configuration {
TEZ_AM_PREFIX + "task.listener.thread-count";
public static final int TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
+ /**
+ * Configuration to limit the counters per AM
+ */
+ public static final String TEZ_AM_COUNTERS_MAX_KEYS = TEZ_AM_PREFIX + "counters.max.keys";
+ public static final int TEZ_AM_COUNTERS_MAX_KEYS_DEFAULT = 1200;
+
+ public static final String TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS =
+ TEZ_AM_PREFIX + "counters.group-name.max.keys";
+ public static final int TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS_DEFAULT = 128;
+
+
+ public static final String TEZ_AM_COUNTERS_NAME_MAX_KEYS =
+ TEZ_AM_PREFIX + "counters.name.max.keys";
+ public static final int TEZ_AM_COUNTERS_NAME_MAX_KEYS_DEFAULT = 64;
+
+
+ public static final String TEZ_AM_COUNTERS_GROUPS_MAX_KEYS =
+ TEZ_AM_PREFIX + "counters.groups.max.keys";
+ public static final int TEZ_AM_COUNTERS_GROUPS_MAX_KEYS_DEFAULT = 500;
+
+
/*
* MR AM Service Authorization
* These are the same as MR which allows Tez to run in secure
@@ -239,6 +267,69 @@ public class TezConfiguration extends Configuration {
public static final String TASK_TIMEOUT = TEZ_TASK_PREFIX + "timeout";
public static final String TASK_HEARTBEAT_TIMEOUT_MS = TEZ_TASK_PREFIX + "heartbeat.timeout-ms";
+
+ /**
+ * Whether to scale down memory requested by each component if the total
+ * exceeds the available JVM memory
+ */
+ @Private
+ @Unstable
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_ENABLED = TEZ_TASK_PREFIX
+ + "scale.task.memory.enabled";
+ public static final boolean TEZ_TASK_SCALE_TASK_MEMORY_ENABLED_DEFAULT = true;
+
+ /**
+ * The allocator to use for initial memory allocation
+ */
+ @Private
+ @Unstable
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS = TEZ_TASK_PREFIX
+ + "scale.task.memory.allocator.class";
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS_DEFAULT =
+ "org.apache.tez.runtime.common.resources.ScalingAllocator";
+
+ /**
+ * The fraction of the JVM memory which will not be considered for allocation.
+ * No defaults, since there are pre-existing defaults based on different scenarios.
+ */
+ @Private
+ @Unstable
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_RESERVE_FRACTION = TEZ_TASK_PREFIX
+ + "scale.task.memory.reserve-fraction";
+ public static final double TEZ_TASK_SCALE_TASK_MEMORY_RESERVE_FRACTION_DEFAULT = 0.3d;
+
+ @Private
+ @Unstable
+ /**
+ * Defines the ProcessTree implementation which will be used to collect resource utilization.
+ */
+ public static final String TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS =
+ TEZ_TASK_PREFIX + "resource.calculator.process-tree.class";
+
+ /**
+ * Fraction of available memory to reserve per input/output. This amount is
+ * removed from the total available pool before allocation and is for factoring in overheads.
+ */
+ @Private
+ @Unstable
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO =
+ TEZ_TASK_PREFIX + "scale.task.memory.additional-reservation.fraction.per-io";
+
+ /**
+ * Max cumulative total reservation for additional IOs.
+ */
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX =
+ TEZ_TASK_PREFIX + "scale.task.memory.additional-reservation.fraction.max";
+ /*
+ * Weighted ratios for individual component types in the RuntimeLibrary.
+ * e.g. PARTITIONED_UNSORTED_OUTPUT:0,UNSORTED_INPUT:1,SORTED_OUTPUT:2,SORTED_MERGED_INPUT:3,PROCESSOR:1,OTHER:1
+ */
+ @Private
+ @Unstable
+ public static final String TEZ_TASK_SCALE_TASK_MEMORY_WEIGHTED_RATIOS =
+ TEZ_TASK_PREFIX + "scale.task.memory.ratios";
+
+
/**
* Configuration to specify whether container should be reused.
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-api/src/test/java/org/apache/tez/common/TestTezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezJobConfig.java b/tez-api/src/test/java/org/apache/tez/common/TestTezJobConfig.java
deleted file mode 100644
index 07364b7..0000000
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezJobConfig.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.tez.common;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.lang.reflect.Field;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.junit.Test;
-
-public class TestTezJobConfig {
-
-
- @Test
- public void testKeySet() throws IllegalAccessException {
- Class<?> c = TezJobConfig.class;
- Set<String> expectedKeys = new HashSet<String>();
- for (Field f : c.getFields()) {
- if (!f.getName().endsWith("DEFAULT") && f.getType() == String.class) {
- expectedKeys.add((String) f.get(null));
- }
- }
-
- Set<String> actualKeySet = TezJobConfig.getRuntimeConfigKeySet();
- for (String key : actualKeySet) {
- if (!expectedKeys.remove(key)) {
- fail("Found unexpected key: " + key + " in key set");
- }
- }
- assertTrue("Missing keys in key set: " + expectedKeys, expectedKeys.size() == 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
index 3dca327..235cc06 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
@@ -26,7 +26,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
/**
- * {@link PolicyProvider} for YARN MapReduce protocols.
+ * {@link PolicyProvider} for YARN Tez client protocols.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index e35faec..f758f6a 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -78,7 +78,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.tez.client.TezClient;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.ProcessorDescriptor;
@@ -98,6 +97,7 @@ import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
import org.apache.tez.mapreduce.partition.MRPartitioner;
import org.apache.tez.mapreduce.processor.map.MapProcessor;
import org.apache.tez.mapreduce.processor.reduce.ReduceProcessor;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
import com.google.common.annotations.VisibleForTesting;
@@ -474,8 +474,8 @@ public class YARNRunner implements ClientProtocol {
// w.r.t Outputs - MAP_OUTPUT_COMPRESS for example)
OrderedPartitionedKVEdgeConfigurer edgeConf =
OrderedPartitionedKVEdgeConfigurer.newBuilder(stageConfs[i - 1].get(
- TezJobConfig.TEZ_RUNTIME_KEY_CLASS),
- stageConfs[i - 1].get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS),
+ TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS),
+ stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS),
MRPartitioner.class.getName(), stageConfs[i - 1])
.configureInput().useLegacyInput().done()
.setFromConfiguration(stageConfs[i - 1]).build();
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 6104b0e..9c8c9b1 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -23,8 +23,8 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.Constants;
public class DeprecatedKeys {
@@ -74,13 +74,13 @@ public class DeprecatedKeys {
// Framework counters, like FILESYSTEM will likely be incompatible since
// they enum key belongs to a different package.
mrParamToDAGParamMap.put(MRJobConfig.COUNTERS_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTERS_MAX_KEY);
+ TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS);
mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTER_GROUP_NAME_MAX_KEY);
+ TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS);
mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTER_NAME_MAX_KEY);
+ TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS);
mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUPS_MAX_KEY,
- TezJobConfig.TEZ_RUNTIME_COUNTER_GROUPS_MAX_KEY);
+ TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS);
}
// TODO TEZAM4 Sometime, make sure this gets loaded by default. Insteaf of the current initialization in MRAppMaster, TezChild.
@@ -91,65 +91,65 @@ public class DeprecatedKeys {
private static void populateMRToTezRuntimeParamMap() {
- registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD, TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD);
+ registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD);
- registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
+ registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
- registerMRToRuntimeKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezJobConfig.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_SORT_SPILL_PERCENT, TezJobConfig.TEZ_RUNTIME_SORT_SPILL_PERCENT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_SORT_SPILL_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT);
- registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_MB, TezJobConfig.TEZ_RUNTIME_IO_SORT_MB);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_MB, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB);
- registerMRToRuntimeKeyTranslation(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT, TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT, TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezJobConfig.TEZ_RUNTIME_COMBINE_MIN_SPILLS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS);
registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, Constants.TEZ_RUNTIME_TASK_MEMORY);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_PARALLEL_COPIES, TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_PARALLEL_COPIES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_FETCH_FAILURES, TezJobConfig.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_FETCH_FAILURES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_NOTIFY_READERROR, TezJobConfig.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_NOTIFY_READERROR, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT, TezJobConfig.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_READ_TIMEOUT, TezJobConfig.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_READ_TIMEOUT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT);
- registerMRToRuntimeKeyTranslation(MRConfig.SHUFFLE_SSL_ENABLED_KEY, TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL);
+ registerMRToRuntimeKeyTranslation(MRConfig.SHUFFLE_SSL_ENABLED_KEY, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT);
- registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MERGE_PERCENT, TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MERGE_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT);
- registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM);
- registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT);
- registerMRToRuntimeKeyTranslation("map.sort.class", TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS);
+ registerMRToRuntimeKeyTranslation("map.sort.class", TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.TEZ_CREDENTIALS_PATH);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezConfiguration.TEZ_CREDENTIALS_PATH);
- registerMRToRuntimeKeyTranslation(MRJobConfig.KEY_COMPARATOR, TezJobConfig.TEZ_RUNTIME_KEY_COMPARATOR_CLASS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.KEY_COMPARATOR, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_KEY_CLASS, TezJobConfig.TEZ_RUNTIME_KEY_CLASS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_KEY_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, TezJobConfig.TEZ_RUNTIME_VALUE_CLASS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezJobConfig.TEZ_RUNTIME_COMPRESS);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
- registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index f0eda1f..38d0711 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.client.TezClientUtils;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.common.security.TokenCache;
@@ -79,6 +78,7 @@ import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRInputUserPayloadProto;
import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto;
import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitsProto;
import org.apache.tez.runtime.api.TezRootInputInitializer;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
@@ -123,7 +123,7 @@ public class MRHelpers {
Configuration baseConf, String stage) {
JobConf jobConf = new JobConf(baseConf);
// Don't clobber explicit tez config.
- if (conf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS) == null) {
+ if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS) == null) {
// If this is set, but the comparator is not set, and their types differ -
// the job will break.
if (conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS) == null) {
@@ -140,7 +140,7 @@ public class MRHelpers {
}
}
- if (conf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS) == null) {
+ if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS) == null) {
if (conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS) == null) {
conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, jobConf
.getMapOutputValueClass().getName());
@@ -619,16 +619,16 @@ public class MRHelpers {
// the AM anyway.
// TODO eventually ACLs
- conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
if (useNewApi) {
if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
- conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
}
} else {
if (conf.get("mapred.combiner.class") != null) {
- conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
index a9f3c3f..3f5ad23 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
@@ -20,9 +20,9 @@ package org.apache.tez.mapreduce.hadoop;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.mapreduce.combine.MRCombiner;
import org.apache.tez.mapreduce.partition.MRPartitioner;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
public class MultiStageMRConfToTezTranslator {
@@ -46,7 +46,7 @@ public class MultiStageMRConfToTezTranslator {
conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
// Setup Tez partitioner class
- conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS,
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
MRPartitioner.class.getName());
// Setup Tez Combiner class if required.
@@ -54,11 +54,11 @@ public class MultiStageMRConfToTezTranslator {
boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
if (useNewApi) {
if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
- conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
}
} else {
if (conf.get("mapred.combiner.class") != null) {
- conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index bbfdc24..d06ef6f 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.Constants;
import org.junit.Test;
@@ -45,22 +45,22 @@ public class TestDeprecatedKeys {
MRHelpers.translateVertexConfToTez(jobConf);
assertEquals(0.4f, jobConf.getFloat(
- TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0f), 0.01f);
+ TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0f), 0.01f);
assertEquals(20000l, jobConf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, 0));
assertEquals(2000,
- jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
+ jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
assertEquals(0.55f, jobConf.getFloat(
- TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0), 0.01f);
+ TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0), 0.01f);
assertEquals(0.60f,
- jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0),
+ jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0),
0.01f);
assertEquals(0.22f,
- jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0),
+ jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0),
0.01f);
assertEquals(true, jobConf.getBoolean(
- TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
+ TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
assertEquals(0.33f,
- jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0),
+ jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0),
0.01f);
}
@@ -74,59 +74,59 @@ public class TestDeprecatedKeys {
jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 200);
- jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 20);
- jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.2f);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 10);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 20);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200);
+ jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 20);
+ jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.2f);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 10);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 20);
jobConf.setInt(Constants.TEZ_RUNTIME_TASK_MEMORY, 10);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 10);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 10);
- jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, true);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 10);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 10);
- jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true);
- jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 10.0f);
- jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 10.0f);
- jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 10.0f);
- jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 10);
- jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, true);
- jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 10.0f);
- jobConf.set(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
- jobConf.set(TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
- jobConf.set(TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 10);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 10);
+ jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, true);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 10);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 10);
+ jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true);
+ jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 10.0f);
+ jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 10.0f);
+ jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 10.0f);
+ jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 10);
+ jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, true);
+ jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 10.0f);
+ jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
+ jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
+ jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
- jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, true);
+ jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true);
MRHelpers.translateVertexConfToTez(jobConf);
- assertEquals(1000, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
- assertEquals(200, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 100));
- assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, false));
- assertEquals(20, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
- assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0));
- assertEquals(20, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 0));
+ assertEquals(1000, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
+ assertEquals(200, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 100));
+ assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false));
+ assertEquals(20, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
+ assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0));
+ assertEquals(20, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 0));
assertEquals(10, jobConf.getInt(Constants.TEZ_RUNTIME_TASK_MEMORY, 0));
- assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 0));
- assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 0));
- assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, false));
- assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 0));
- assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 0));
- assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, false));
- assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.0f);
- assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.0f);
- assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.0f);
- assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0));
- assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
- assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), 0.0f);
- assertEquals("DefaultSorter", jobConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
- assertEquals("groupComparator", jobConf.get(TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, ""));
- assertEquals("SecondaryComparator", jobConf.get(TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, ""));
- assertEquals("DefaultSorter", jobConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
- assertTrue(jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, false));
+ assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 0));
+ assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 0));
+ assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, false));
+ assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 0));
+ assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 0));
+ assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, false));
+ assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.0f);
+ assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.0f);
+ assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.0f);
+ assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0));
+ assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
+ assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), 0.0f);
+ assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
+ assertEquals("groupComparator", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, ""));
+ assertEquals("SecondaryComparator", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, ""));
+ assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
+ assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false));
assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
index 5ffee56..f017436 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.tez.common.MRFrameworkConfigs;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.InputDescriptor;
@@ -47,6 +46,7 @@ import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
import org.apache.tez.runtime.api.TezInputContext;
import org.apache.tez.runtime.api.impl.InputSpec;
import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.Constants;
import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
import org.apache.tez.runtime.library.common.sort.impl.IFile;
@@ -90,7 +90,7 @@ public class TestMapProcessor {
Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class,
TezTaskOutput.class);
- job.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
+ job.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
job.setNumReduceTasks(1);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
index b1b865d..c1798d3 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.security.token.Token;
import org.apache.tez.common.MRFrameworkConfigs;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -58,6 +57,7 @@ import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
import org.apache.tez.runtime.api.impl.InputSpec;
import org.apache.tez.runtime.api.impl.OutputSpec;
import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.Constants;
import org.apache.tez.runtime.library.common.task.local.output.TezLocalTaskOutputFiles;
import org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput;
@@ -101,7 +101,7 @@ public class TestReduceProcessor {
Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class,
TezTaskOutput.class);
- job.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
+ job.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
job.setNumReduceTasks(1);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
index 6d2f852..2efafab 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
@@ -31,7 +31,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezEntityDescriptor;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.runtime.api.MemoryUpdateCallback;
@@ -74,8 +74,8 @@ public class MemoryDistributor {
*/
public MemoryDistributor(int numTotalInputs, int numTotalOutputs, Configuration conf) {
this.conf = conf;
- isEnabled = conf.getBoolean(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED,
- TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED_DEFAULT);
+ isEnabled = conf.getBoolean(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ENABLED,
+ TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ENABLED_DEFAULT);
this.numTotalInputs = numTotalInputs;
@@ -93,7 +93,7 @@ public class MemoryDistributor {
* Used by the Tez framework to request memory on behalf of user requests.
*/
public void requestMemory(long requestSize, MemoryUpdateCallback callback,
- TezTaskContext taskContext, TezEntityDescriptor descriptor) {
+ TezTaskContext taskContext, TezEntityDescriptor<?> descriptor) {
registerRequest(requestSize, callback, taskContext, descriptor);
}
@@ -119,8 +119,8 @@ public class MemoryDistributor {
}
});
} else {
- String allocatorClassName = conf.get(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ALLOCATOR_CLASS,
- TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ALLOCATOR_CLASS_DEFAULT);
+ String allocatorClassName = conf.get(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS,
+ TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS_DEFAULT);
LOG.info("Using Allocator class: " + allocatorClassName);
InitialMemoryAllocator allocator = ReflectionUtils.createClazzInstance(allocatorClassName);
allocator.setConf(conf);
@@ -154,7 +154,7 @@ public class MemoryDistributor {
}
private long registerRequest(long requestSize, MemoryUpdateCallback callback,
- TezTaskContext entityContext, TezEntityDescriptor descriptor) {
+ TezTaskContext entityContext, TezEntityDescriptor<?> descriptor) {
Preconditions.checkArgument(requestSize >= 0);
Preconditions.checkNotNull(callback);
Preconditions.checkNotNull(entityContext);
@@ -211,7 +211,7 @@ public class MemoryDistributor {
private final InitialMemoryRequestContext requestContext;
public RequestorInfo(TezTaskContext taskContext, long requestSize,
- final MemoryUpdateCallback callback, TezEntityDescriptor descriptor) {
+ final MemoryUpdateCallback callback, TezEntityDescriptor<?> descriptor) {
InitialMemoryRequestContext.ComponentType type;
String componentVertexName;
if (taskContext instanceof TezInputContext) {
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java
index 8f08b6e..8ded69d 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java
@@ -24,9 +24,8 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.dag.api.TezConfiguration;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
@@ -36,9 +35,6 @@ public class ScalingAllocator implements InitialMemoryAllocator {
private static final Log LOG = LogFactory.getLog(ScalingAllocator.class);
- @VisibleForTesting
- static final double DEFAULT_RESERVE_FRACTION = 0.3d;
-
private Configuration conf;
@Override
@@ -53,8 +49,8 @@ public class ScalingAllocator implements InitialMemoryAllocator {
}
// Take a certain amount of memory away for general usage.
- double reserveFraction = conf.getDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_RESERVE_FRACTION,
- DEFAULT_RESERVE_FRACTION);
+ double reserveFraction = conf.getDouble(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_RESERVE_FRACTION,
+ TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_RESERVE_FRACTION_DEFAULT);
Preconditions.checkState(reserveFraction >= 0.0d && reserveFraction <= 1.0d);
availableForAllocation = (long) (availableForAllocation - (reserveFraction * availableForAllocation));
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java
index 1972535..cc57887 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java
@@ -29,9 +29,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.TezConfiguration;
/**
@@ -139,7 +139,7 @@ public class TaskCounterUpdater {
private void initResourceCalculatorPlugin() {
Class<? extends ResourceCalculatorProcessTree> clazz = this.conf.getClass(
- TezJobConfig.TEZ_RUNTIME_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
+ TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
ResourceCalculatorProcessTree.class);
pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
index 18e5766..522ec23 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java
@@ -24,10 +24,10 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.runtime.api.MemoryUpdateCallback;
import org.apache.tez.runtime.api.TezInputContext;
import org.apache.tez.runtime.api.TezOutputContext;
@@ -41,8 +41,8 @@ public class TestMemoryDistributor {
@Before
public void setup() {
- conf.setBoolean(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED, true);
- conf.set(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ALLOCATOR_CLASS,
+ conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ENABLED, true);
+ conf.set(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ALLOCATOR_CLASS,
ScalingAllocator.class.getName());
}
@@ -151,7 +151,7 @@ public class TestMemoryDistributor {
public void testScalingDisabled() {
// Real world values
Configuration conf = new Configuration(this.conf);
- conf.setBoolean(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ENABLED, false);
+ conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_ENABLED, false);
MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
dist.setJvmMemory(207093760l);
@@ -177,7 +177,7 @@ public class TestMemoryDistributor {
@Test(timeout = 5000)
public void testReserveFractionConfigured() {
Configuration conf = new Configuration(this.conf);
- conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_RESERVE_FRACTION, 0.5d);
+ conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_TASK_MEMORY_RESERVE_FRACTION, 0.5d);
MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
dist.setJvmMemory(10000l);
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java
index 0647262..1444413 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java
@@ -18,14 +18,13 @@
package org.apache.tez.runtime.library.api;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
/**
* {@link Partitioner} is used by the TEZ framework to partition output
* key/value pairs.
*
* <b>Partitioner Initialization</b></p> The Partitioner class is picked up
- * using the TEZ_RUNTIME_PARTITIONER_CLASS attribute in {@link TezJobConfig}
+ * using the TEZ_RUNTIME_PARTITIONER_CLASS attribute in {@link TezRuntimeConfiguration}
*
* TODO NEWTEZ Change construction to first check for a Constructor with a bytep[] payload
*