You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2015/09/01 22:55:57 UTC
[2/2] tez git commit: TEZ-2294. Add tez-site-template.xml with
description of config properties. (hitesh)
TEZ-2294. Add tez-site-template.xml with description of config properties. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/50df8651
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/50df8651
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/50df8651
Branch: refs/heads/master
Commit: 50df8651f27d8bc8d463cf8c8cd70ceea3e6af4f
Parents: 5cb0079
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Sep 1 13:55:16 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Sep 1 13:55:41 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
docs/pom.xml | 1 +
pom.xml | 61 +++--
tez-api/pom.xml | 33 ++-
.../common/annotation/ConfigurationClass.java | 31 +++
.../annotation/ConfigurationProperty.java | 36 +++
.../apache/tez/dag/api/TezConfiguration.java | 126 +++++++++-
tez-dist/pom.xml | 5 +
tez-runtime-library/pom.xml | 32 +++
.../library/api/TezRuntimeConfiguration.java | 67 ++++--
tez-tools/pom.xml | 2 +
.../tez-javadoc-tools/findbugs-exclude.xml | 22 ++
tez-tools/tez-javadoc-tools/pom.xml | 92 ++++++++
.../javadoc/doclet/ConfigStandardDoclet.java | 233 +++++++++++++++++++
.../apache/tez/tools/javadoc/model/Config.java | 40 ++++
.../tez/tools/javadoc/model/ConfigProperty.java | 46 ++++
.../tez/tools/javadoc/util/HtmlWriter.java | 161 +++++++++++++
.../apache/tez/tools/javadoc/util/Writer.java | 39 ++++
.../tez/tools/javadoc/util/XmlWriter.java | 96 ++++++++
19 files changed, 1083 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 66fe000..3c7d0bd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.8.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2294. Add tez-site-template.xml with description of config properties.
TEZ-2757. Fix download links for Tez releases.
TEZ-2742. VertexImpl.finished() terminationCause hides member var of the
same name
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index b76f9ba..df6f3ef 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -444,6 +444,7 @@
<artifactId>maven-project-info-reports-plugin</artifactId>
<configuration>
<customBundle>${project.basedir}/src/site/custom/project-info-report.properties</customBundle>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
</configuration>
<reportSets>
<reportSet>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fe996d0..d1d5fe6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -160,6 +160,11 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>tez-tools</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-plugins</artifactId>
<version>${project.version}</version>
</dependency>
@@ -885,6 +890,14 @@
</lifecycleMappingMetadata>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
@@ -1134,6 +1147,37 @@
</build>
</profile>
<profile>
+ <id>jdk1.7</id>
+ <activation>
+ <jdk>1.7</jdk>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ <version>1.7</version>
+ <scope>system</scope>
+ <systemPath>${java.home}/../lib/tools.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>jdk1.8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ <version>1.8</version>
+ <scope>system</scope>
+ <systemPath>${java.home}/../lib/tools.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
@@ -1187,21 +1231,12 @@
<frontend-maven-plugin.version>0.0.22</frontend-maven-plugin.version>
</properties>
</profile>
-
</profiles>
<reporting>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>2.7</version>
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${javadoc-maven-plugin.version}</version>
<reportSets>
@@ -1224,14 +1259,6 @@
</reportSet>
</reportSets>
</plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <version>${findbugs-maven-plugin.version}</version>
- <configuration>
- <excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- </plugin>
</plugins>
</reporting>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index 1baec8f..b3eb97c 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -27,7 +27,7 @@
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -177,4 +177,35 @@
</plugins>
</build>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <reportSets>
+ <reportSet>
+ <id>configgen</id>
+ <configuration>
+ <doclet>org.apache.tez.tools.javadoc.doclet.ConfigStandardDoclet</doclet>
+ <docletArtifacts>
+ <docletArtifact>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-javadoc-tools</artifactId>
+ <version>${project.version}</version>
+ </docletArtifact>
+ </docletArtifacts>
+ <destDir>apidocs/configs</destDir>
+ <name>TezConfigGenerator</name>
+ <description>Tez Configuration documentation</description>
+ </configuration>
+ <reports>
+ <report>javadoc</report>
+ <report>aggregate</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ </plugins>
+ </reporting>
+
</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java
new file mode 100644
index 0000000..6b8fdaf
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+@Target(ElementType.TYPE)
+public @interface ConfigurationClass {
+
+ public String templateFileName();
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java
new file mode 100644
index 0000000..f7d37e2
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ConfigurationProperty {
+
+ public boolean mandatory() default false;
+ public String type() default "string";
+ public String[] validValues() default {};
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 54d490b..4d12740 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -23,6 +23,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import org.apache.tez.common.annotation.ConfigurationClass;
+import org.apache.tez.common.annotation.ConfigurationProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -39,8 +41,11 @@ import com.google.common.annotations.VisibleForTesting;
* Defines the configurations for Tez. These configurations are typically specified in
* tez-site.xml on the client machine where TezClient is used to launch the Tez application.
* tez-site.xml is expected to be picked up from the classpath of the client process.
+ * @see <a href="../../../../../configs/TezConfiguration.html">Detailed Configuration Information</a>
+ * @see <a href="../../../../../configs/tez-default-template.xml">XML-based Config Template</a>
*/
@Public
+@ConfigurationClass(templateFileName = "tez-default-template.xml")
public class TezConfiguration extends Configuration {
public final static String TEZ_SITE_XML = "tez-site.xml";
@@ -122,6 +127,7 @@ public class TezConfiguration extends Configuration {
* artifacts that it creates within the specified staging dir. Does not affect any user data.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE = TEZ_AM_PREFIX +
"staging.scratch-data.auto-delete";
public static final boolean TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE_DEFAULT = true;
@@ -130,6 +136,7 @@ public class TezConfiguration extends Configuration {
* String value. Specifies a directory where Tez can create temporary job artifacts.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/"
+ System.getProperty("user.name") + "/tez/staging";
@@ -139,6 +146,7 @@ public class TezConfiguration extends Configuration {
* Path to a credentials file (with serialized credentials) located on the local file system.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_CREDENTIALS_PATH = TEZ_PREFIX + "credentials.path";
@Private
@@ -164,6 +172,7 @@ public class TezConfiguration extends Configuration {
* mode is recommended. If session mode is enabled then container reuse is recommended.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session";
public static final boolean TEZ_AM_SESSION_MODE_DEFAULT = false;
@@ -184,6 +193,7 @@ public class TezConfiguration extends Configuration {
* org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
* */
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX + "log.level";
public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
@@ -204,6 +214,7 @@ public class TezConfiguration extends Configuration {
* org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
* */
@ConfigurationScope(Scope.VERTEX)
+ @ConfigurationProperty
public static final String TEZ_TASK_LOG_LEVEL = TEZ_TASK_PREFIX + "log.level";
public static final String TEZ_TASK_LOG_LEVEL_DEFAULT = "INFO";
@@ -240,6 +251,7 @@ public class TezConfiguration extends Configuration {
* this value must be appropriately chosen. Defaults to the safe choice of true.
*/
@ConfigurationScope(Scope.DAG)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS =
TEZ_AM_PREFIX + "commit-all-outputs-on-dag-success";
public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true;
@@ -260,6 +272,7 @@ public class TezConfiguration extends Configuration {
* be overridden per job.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS =
TEZ_AM_PREFIX + "launch.cluster-default.cmd-opts";
public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
@@ -271,6 +284,7 @@ public class TezConfiguration extends Configuration {
* Tez can determine them automatically.
* */
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX + "launch.cmd-opts";
public static final String TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT =
"-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC";
@@ -282,6 +296,7 @@ public class TezConfiguration extends Configuration {
* be overridden per job.
*/
@ConfigurationScope(Scope.AM) // TODO DAG/Vertex level
+ @ConfigurationProperty
public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS =
TEZ_TASK_PREFIX + "launch.cluster-default.cmd-opts";
public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
@@ -293,6 +308,7 @@ public class TezConfiguration extends Configuration {
* so that Tez can determine them automatically.
*/
@ConfigurationScope(Scope.AM) // TODO DAG/Vertex level
+ @ConfigurationProperty
public static final String TEZ_TASK_LAUNCH_CMD_OPTS = TEZ_TASK_PREFIX
+ "launch.cmd-opts";
public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT =
@@ -308,6 +324,7 @@ public class TezConfiguration extends Configuration {
* on container memory size. Value should be greater than 0 and less than 1.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="float")
public static final String TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION =
TEZ_PREFIX + "container.max.java.heap.fraction";
public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT = 0.8;
@@ -324,6 +341,7 @@ public class TezConfiguration extends Configuration {
* These get added to the app master environment prior to launching it.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_LAUNCH_ENV = TEZ_AM_PREFIX
+ "launch.env";
public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT;
@@ -336,12 +354,14 @@ public class TezConfiguration extends Configuration {
* These get added to the task environment prior to launching it.
*/
@ConfigurationScope(Scope.VERTEX)
+ @ConfigurationProperty
public static final String TEZ_TASK_LAUNCH_ENV = TEZ_TASK_PREFIX
+ "launch.env";
public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT;
@Private
@ConfigurationScope(Scope.DAG)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION = TEZ_PREFIX +
"cancel.delegation.tokens.on.completion";
public static final boolean TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT = true;
@@ -358,6 +378,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_TASK_LISTENER_THREAD_COUNT =
TEZ_AM_PREFIX + "task.listener.thread-count";
public static final int TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
@@ -370,6 +391,7 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_COUNTERS_MAX = TEZ_PREFIX + "counters.max";
public static final int TEZ_COUNTERS_MAX_DEFAULT = 1200;
@@ -380,6 +402,7 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_COUNTERS_MAX_GROUPS = TEZ_PREFIX + "counters.max.groups";
public static final int TEZ_COUNTERS_MAX_GROUPS_DEFAULT = 500;
@@ -390,6 +413,7 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH =
TEZ_PREFIX + "counters.counter-name.max-length";
public static final int TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH_DEFAULT = 64;
@@ -401,6 +425,7 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH =
TEZ_PREFIX + "counters.group-name.max-length";
public static final int TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH_DEFAULT = 256;
@@ -411,6 +436,7 @@ public class TezConfiguration extends Configuration {
* when some tasks are running slower due bad/slow machines
*/
@ConfigurationScope(Scope.VERTEX) // TODO Verify the vertex speculation, TEZ-1788
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_SPECULATION_ENABLED = TEZ_AM_PREFIX + "speculation.enabled";
public static final boolean TEZ_AM_SPECULATION_ENABLED_DEFAULT = false;
@@ -420,7 +446,8 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_LEGACY_SPECULATIVE_SLOWTASK_THRESHOLD =
+ @ConfigurationProperty(type="float")
+ public static final String TEZ_AM_LEGACY_SPECULATIVE_SLOWTASK_THRESHOLD =
TEZ_AM_PREFIX + "legacy.speculative.slowtask.threshold";
/**
@@ -428,6 +455,7 @@ public class TezConfiguration extends Configuration {
* master. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT =
TEZ_AM_PREFIX + "containerlauncher.thread-count-limit";
@@ -439,6 +467,7 @@ public class TezConfiguration extends Configuration {
* Int value. Specifies the number of task failures on a node before the node is considered faulty.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_MAX_TASK_FAILURES_PER_NODE = TEZ_AM_PREFIX
+ "maxtaskfailures.per.node";
public static final int TEZ_AM_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 10;
@@ -450,7 +479,8 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_MAX_APP_ATTEMPTS = TEZ_AM_PREFIX +
+ @ConfigurationProperty(type="integer")
+ public static final String TEZ_AM_MAX_APP_ATTEMPTS = TEZ_AM_PREFIX +
"max.app.attempts";
public static final int TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT = 2;
@@ -459,6 +489,7 @@ public class TezConfiguration extends Configuration {
* This does not count killed attempts. Task failure results in DAG failure.
*/
@ConfigurationScope(Scope.VERTEX)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_TASK_MAX_FAILED_ATTEMPTS =
TEZ_AM_PREFIX + "task.max.failed.attempts";
public static final int TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT = 4;
@@ -468,6 +499,7 @@ public class TezConfiguration extends Configuration {
* will not be used to execute tasks.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_NODE_BLACKLISTING_ENABLED = TEZ_AM_PREFIX
+ "node-blacklisting.enabled";
public static final boolean TEZ_AM_NODE_BLACKLISTING_ENABLED_DEFAULT = true;
@@ -478,12 +510,14 @@ public class TezConfiguration extends Configuration {
* temporary surges in failures (e.g. due to network outages).
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_AM_PREFIX
+ "node-blacklisting.ignore-threshold-node-percent";
public static final int TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;
/** Int value. Number of threads to handle client RPC requests. Expert level setting.*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_CLIENT_THREAD_COUNT =
TEZ_AM_PREFIX + "client.am.thread-count";
public static final int TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT = 1;
@@ -494,6 +528,7 @@ public class TezConfiguration extends Configuration {
* For example 50000-50050,50100-50200
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_CLIENT_AM_PORT_RANGE =
TEZ_AM_PREFIX + "client.am.port-range";
@@ -510,18 +545,21 @@ public class TezConfiguration extends Configuration {
* String value. The class to be used for DAG Scheduling. Expert level setting.
*/
@ConfigurationScope(Scope.DAG)
+ @ConfigurationProperty
public static final String TEZ_AM_DAG_SCHEDULER_CLASS = TEZ_AM_PREFIX + "dag.scheduler.class";
public static final String TEZ_AM_DAG_SCHEDULER_CLASS_DEFAULT =
"org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder";
/** Int value. The amount of memory in MB to be used by the AppMaster */
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_RESOURCE_MEMORY_MB = TEZ_AM_PREFIX
+ "resource.memory.mb";
public static final int TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT = 1024;
/** Int value. The number of virtual cores to be used by the app master */
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_RESOURCE_CPU_VCORES = TEZ_AM_PREFIX
+ "resource.cpu.vcores";
public static final int TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT = 1;
@@ -530,6 +568,7 @@ public class TezConfiguration extends Configuration {
* all vertices. Setting it to the same value for all tasks is helpful for container reuse and
* thus good for performance typically. */
@ConfigurationScope(Scope.DAG) // TODO vertex level
+ @ConfigurationProperty(type="integer")
public static final String TEZ_TASK_RESOURCE_MEMORY_MB = TEZ_TASK_PREFIX
+ "resource.memory.mb";
public static final int TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT = 1024;
@@ -538,6 +577,7 @@ public class TezConfiguration extends Configuration {
* Int value. The number of virtual cores to be used by tasks.
*/
@ConfigurationScope(Scope.DAG) // TODO vertex level
+ @ConfigurationProperty(type="integer")
public static final String TEZ_TASK_RESOURCE_CPU_VCORES = TEZ_TASK_PREFIX
+ "resource.cpu.vcores";
public static final int TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT = 1;
@@ -548,6 +588,7 @@ public class TezConfiguration extends Configuration {
* help in scaling up. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX = TEZ_AM_PREFIX
+ "am-rm.heartbeat.interval-ms.max";
public static final int TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX_DEFAULT = 1000;
@@ -558,6 +599,7 @@ public class TezConfiguration extends Configuration {
* number of concurrent tasks. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX = TEZ_TASK_PREFIX
+ "get-task.sleep.interval-ms.max";
public static final int TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT = 200;
@@ -568,6 +610,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS = TEZ_TASK_PREFIX
+ "am.heartbeat.interval-ms.max";
public static final int TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT = 100;
@@ -578,6 +621,7 @@ public class TezConfiguration extends Configuration {
* counters. Improves AM scalability. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS = TEZ_TASK_PREFIX
+ "am.heartbeat.counter.interval-ms.max";
public static final int TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT =
@@ -588,6 +632,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT = TEZ_TASK_PREFIX
+ "max-events-per-heartbeat";
public static final int TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT = 500;
@@ -600,6 +645,7 @@ public class TezConfiguration extends Configuration {
@Unstable
@Private
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_TASK_GENERATE_COUNTERS_PER_IO = TEZ_TASK_PREFIX
+ "generate.counters.per.io";
@Private
@@ -611,6 +657,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TASK_HEARTBEAT_TIMEOUT_MS = TEZ_TASK_PREFIX + "timeout-ms";
public static final int TASK_HEARTBEAT_TIMEOUT_MS_DEFAULT = 5 * 60 * 1000;
@@ -620,6 +667,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TASK_HEARTBEAT_TIMEOUT_CHECK_MS = TEZ_TASK_PREFIX + "heartbeat.timeout.check-ms";
public static final int TASK_HEARTBEAT_TIMEOUT_CHECK_MS_DEFAULT = 30 * 1000;
@@ -631,6 +679,7 @@ public class TezConfiguration extends Configuration {
@Private
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_TASK_SCALE_MEMORY_ENABLED = TEZ_TASK_PREFIX
+ "scale.memory.enabled";
@Private
@@ -642,6 +691,7 @@ public class TezConfiguration extends Configuration {
@Private
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS = TEZ_TASK_PREFIX
+ "scale.memory.allocator.class";
@Private
@@ -655,6 +705,7 @@ public class TezConfiguration extends Configuration {
@Private
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="double")
public static final String TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION = TEZ_TASK_PREFIX
+ "scale.memory.reserve-fraction";
@Private
@@ -667,6 +718,7 @@ public class TezConfiguration extends Configuration {
@Private
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="float")
public static final String TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO =
TEZ_TASK_PREFIX + "scale.memory.additional-reservation.fraction.per-io";
@@ -676,6 +728,7 @@ public class TezConfiguration extends Configuration {
* Max cumulative total reservation for additional IOs.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="float")
public static final String TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX =
TEZ_TASK_PREFIX + "scale.memory.additional-reservation.fraction.max";
/*
@@ -686,6 +739,7 @@ public class TezConfiguration extends Configuration {
@Private
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS =
TEZ_TASK_PREFIX + "scale.memory.ratios";
@@ -695,6 +749,7 @@ public class TezConfiguration extends Configuration {
* Defines the ProcessTree implementation which will be used to collect resource utilization.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS =
TEZ_TASK_PREFIX + "resource.calculator.process-tree.class";
@@ -704,6 +759,7 @@ public class TezConfiguration extends Configuration {
* This improves performance by not incurring recurring launch overheads.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_CONTAINER_REUSE_ENABLED = TEZ_AM_PREFIX
+ "container.reuse.enabled";
public static final boolean TEZ_AM_CONTAINER_REUSE_ENABLED_DEFAULT = true;
@@ -713,6 +769,7 @@ public class TezConfiguration extends Configuration {
* enabled.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED =
TEZ_AM_PREFIX + "container.reuse.rack-fallback.enabled";
public static final boolean
@@ -724,6 +781,7 @@ public class TezConfiguration extends Configuration {
* volume being read from the primary data sources.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED =
TEZ_AM_PREFIX + "container.reuse.non-local-fallback.enabled";
public static final boolean
@@ -734,6 +792,7 @@ public class TezConfiguration extends Configuration {
* of locality. NODE -> RACK -> NON_LOCAL. Delay scheduling parameter. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="long")
public static final String
TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS =
TEZ_AM_PREFIX + "container.reuse.locality.delay-allocation-millis";
@@ -745,7 +804,8 @@ public class TezConfiguration extends Configuration {
* reuse is enabled. Set to -1 to never release idle containers (not recommended).
*/
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS =
+ @ConfigurationProperty(type="integer")
+ public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS =
TEZ_AM_PREFIX + "container.idle.release-timeout-min.millis";
public static final long
TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT = 5000l;
@@ -761,7 +821,8 @@ public class TezConfiguration extends Configuration {
* creates a graceful reduction in the amount of idle resources held
*/
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS =
+ @ConfigurationProperty(type="long")
+ public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS =
TEZ_AM_PREFIX + "container.idle.release-timeout-max.millis";
public static final long
TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT = 10000l;
@@ -772,7 +833,8 @@ public class TezConfiguration extends Configuration {
* of containers to provide fast response times for the next DAG.
*/
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_SESSION_MIN_HELD_CONTAINERS =
+ @ConfigurationProperty(type="integer")
+ public static final String TEZ_AM_SESSION_MIN_HELD_CONTAINERS =
TEZ_AM_PREFIX + "session.min.held-containers";
public static final int TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT = 0;
@@ -795,7 +857,8 @@ public class TezConfiguration extends Configuration {
* setting.
*/
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_PREEMPTION_PERCENTAGE =
+ @ConfigurationProperty(type="integer")
+ public static final String TEZ_AM_PREEMPTION_PERCENTAGE =
TEZ_AM_PREFIX + "preemption.percentage";
public static final int TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT = 10;
@@ -805,7 +868,8 @@ public class TezConfiguration extends Configuration {
* RM can act on the released resources and assign new ones to us. Expert level setting.
*/
@ConfigurationScope(Scope.AM)
- public static final String TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS =
+ @ConfigurationProperty(type="integer")
+ public static final String TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS =
TEZ_AM_PREFIX + "preemption.heartbeats-between-preemptions";
public static final int TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT = 3;
@@ -831,6 +895,7 @@ public class TezConfiguration extends Configuration {
* </ol>
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_LIB_URIS = TEZ_PREFIX + "lib.uris";
/**
@@ -846,6 +911,7 @@ public class TezConfiguration extends Configuration {
*
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AUX_URIS = TEZ_PREFIX + "aux.uris";
/**
@@ -853,7 +919,9 @@ public class TezConfiguration extends Configuration {
* raw Tez application where classpath is propagated with application
* via {@link LocalResource}s. This is mainly useful for developer/debugger scenarios.
*/
+ @Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_IGNORE_LIB_URIS = TEZ_PREFIX + "ignore.lib.uris";
/**
@@ -863,6 +931,7 @@ public class TezConfiguration extends Configuration {
* tez-deployment which contains the hadoop libraries.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_USE_CLUSTER_HADOOP_LIBS = TEZ_PREFIX + "use.cluster.hadoop-libs";
public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false;
@@ -874,6 +943,7 @@ public class TezConfiguration extends Configuration {
* specified.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX =
TEZ_PREFIX + "cluster.additional.classpath.prefix";
@@ -881,6 +951,7 @@ public class TezConfiguration extends Configuration {
* Session-related properties
*/
@Private
+ @ConfigurationProperty
public static final String TEZ_SESSION_PREFIX =
TEZ_PREFIX + "session.";
@@ -891,6 +962,7 @@ public class TezConfiguration extends Configuration {
* applicable. Otherwise increase the timeout (set to -1 for infinity. Not recommended)
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_SESSION_CLIENT_TIMEOUT_SECS =
TEZ_SESSION_PREFIX + "client.timeout.secs";
public static final int TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT =
@@ -901,6 +973,7 @@ public class TezConfiguration extends Configuration {
* shutting down. Only relevant in session mode.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS =
TEZ_SESSION_PREFIX + "am.dag.submit.timeout.secs";
public static final int TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT =
@@ -910,6 +983,7 @@ public class TezConfiguration extends Configuration {
* String value. The queue name for all jobs being submitted from a given client.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name";
/**
@@ -923,6 +997,7 @@ public class TezConfiguration extends Configuration {
* Boolean value. Generate debug artifacts like DAG plan in text format.
*/
@ConfigurationScope(Scope.DAG)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_GENERATE_DEBUG_ARTIFACTS =
TEZ_PREFIX + "generate.debug.artifacts";
public static final boolean TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT = false;
@@ -940,6 +1015,7 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST = TEZ_PREFIX + "task-specific" +
".launch.cmd-opts.list";
@@ -952,6 +1028,7 @@ public class TezConfiguration extends Configuration {
*/
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS = TEZ_PREFIX + "task-specific" +
".launch.cmd-opts";
@@ -973,6 +1050,7 @@ public class TezConfiguration extends Configuration {
* */
@Unstable
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_TASK_SPECIFIC_LOG_LEVEL = TEZ_PREFIX + "task-specific" + ".log.level";
/**
@@ -980,6 +1058,7 @@ public class TezConfiguration extends Configuration {
* Specify the class to use for logging history data
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_HISTORY_LOGGING_SERVICE_CLASS =
TEZ_PREFIX + "history.logging.service.class";
@@ -992,6 +1071,7 @@ public class TezConfiguration extends Configuration {
* used for {@link TezConfiguration#TEZ_HISTORY_LOGGING_SERVICE_CLASS}
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_SIMPLE_HISTORY_LOGGING_DIR =
TEZ_PREFIX + "simple.history.logging.dir";
@@ -1000,6 +1080,7 @@ public class TezConfiguration extends Configuration {
* logging gets disabled. The job continues to run after this.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_SIMPLE_HISTORY_LOGGING_MAX_ERRORS =
TEZ_PREFIX + "simple.history.max.errors";
public static final int TEZ_SIMPLE_HISTORY_LOGGING_MAX_ERRORS_DEFAULT = 10;
@@ -1009,6 +1090,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="long")
public static final String YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS =
TEZ_PREFIX + "yarn.ats.event.flush.timeout.millis";
public static final long YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT =
@@ -1019,6 +1101,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String YARN_ATS_MAX_EVENTS_PER_BATCH =
TEZ_PREFIX + "yarn.ats.max.events.per.batch";
public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5;
@@ -1029,11 +1112,13 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String YARN_ATS_MAX_POLLING_TIME_PER_EVENT = TEZ_PREFIX
+ "yarn.ats.max.polling.time.per.event.millis";
public static final int YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT = 10;
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String YARN_ATS_ACL_DOMAINS_AUTO_CREATE = TEZ_PREFIX
+ "yarn.ats.acl.domains.auto-create";
public static final boolean YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT = true;
@@ -1050,6 +1135,7 @@ public class TezConfiguration extends Configuration {
* incomplete DAGs from the previous instance of the app master.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String DAG_RECOVERY_ENABLED =
TEZ_PREFIX + "dag.recovery.enabled";
public static final boolean DAG_RECOVERY_ENABLED_DEFAULT = true;
@@ -1059,6 +1145,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String DAG_RECOVERY_FILE_IO_BUFFER_SIZE =
TEZ_PREFIX + "dag.recovery.io.buffer.size";
public static final int DAG_RECOVERY_FILE_IO_BUFFER_SIZE_DEFAULT = 8192;
@@ -1067,6 +1154,7 @@ public class TezConfiguration extends Configuration {
* Int value. Number of recovery events to buffer before flushing them to the recovery log.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String DAG_RECOVERY_MAX_UNFLUSHED_EVENTS =
TEZ_PREFIX + "dag.recovery.max.unflushed.events";
public static final int DAG_RECOVERY_MAX_UNFLUSHED_EVENTS_DEFAULT = 100;
@@ -1075,6 +1163,7 @@ public class TezConfiguration extends Configuration {
* Int value. Interval, in seconds, between flushing recovery data to the recovery log.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String DAG_RECOVERY_FLUSH_INTERVAL_SECS =
TEZ_PREFIX + "dag.recovery.flush.interval.secs";
public static final int DAG_RECOVERY_FLUSH_INTERVAL_SECS_DEFAULT = 30;
@@ -1084,6 +1173,7 @@ public class TezConfiguration extends Configuration {
* the app master. Primarily used for debugging.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_LOCAL_MODE =
TEZ_PREFIX + "local.mode";
@@ -1094,6 +1184,7 @@ public class TezConfiguration extends Configuration {
*/
@Private
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_INLINE_TASK_EXECUTION_ENABLED =
TEZ_AM_PREFIX + "inline.task.execution.enabled";
@@ -1108,6 +1199,7 @@ public class TezConfiguration extends Configuration {
* The maximium number of tasks running in parallel within the app master process.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="integer")
public static final String TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS =
TEZ_AM_PREFIX + "inline.task.execution.max-tasks";
@@ -1133,6 +1225,7 @@ public class TezConfiguration extends Configuration {
* Boolean value. Configuration to enable/disable ACL checks.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_ACLS_ENABLED = TEZ_AM_PREFIX + "acls.enabled";
public static final boolean TEZ_AM_ACLS_ENABLED_DEFAULT = true;
@@ -1144,6 +1237,7 @@ public class TezConfiguration extends Configuration {
* groups
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_VIEW_ACLS = TEZ_AM_PREFIX + "view-acls";
/**
@@ -1154,13 +1248,16 @@ public class TezConfiguration extends Configuration {
* groups
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_MODIFY_ACLS = TEZ_AM_PREFIX + "modify-acls";
/**
* Boolean value.
* Disable version check between client and AM/DAG. Default false.
*/
+ @Private
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_DISABLE_CLIENT_VERSION_CHECK = TEZ_AM_PREFIX
+ "disable.client-version-check";
public static final boolean TEZ_AM_DISABLE_CLIENT_VERSION_CHECK_DEFAULT = false;
@@ -1169,7 +1266,9 @@ public class TezConfiguration extends Configuration {
* Boolean value.
* Allow disabling of Timeline Domains even if Timeline is being used.
*/
+ @Private
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS = TEZ_PREFIX
+ "allow.disabled.timeline-domains";
public static final boolean TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT = false;
@@ -1189,6 +1288,7 @@ public class TezConfiguration extends Configuration {
* http://uihost:9001/#/tez-app/application_1421880306565_0001/
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE = TEZ_AM_PREFIX
+ "tez-ui.history-url.template";
public static final String TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT =
@@ -1202,6 +1302,7 @@ public class TezConfiguration extends Configuration {
* be specified.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
public static final String TEZ_HISTORY_URL_BASE = TEZ_PREFIX
+ "tez-ui.history-url.base";
@@ -1211,6 +1312,7 @@ public class TezConfiguration extends Configuration {
* updates for running application.
*/
@ConfigurationScope(Scope.AM)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_AM_WEBSERVICE_ENABLE = TEZ_AM_PREFIX
+ "tez-ui.webservice.enable";
public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true;
@@ -1240,6 +1342,7 @@ public class TezConfiguration extends Configuration {
* Status Poll interval in Milliseconds used when getting DAG status with timeout.
*/
@ConfigurationScope(Scope.DAG)
+ @ConfigurationProperty(type="long")
public static final String TEZ_DAG_STATUS_POLLINTERVAL_MS = TEZ_PREFIX
+ "dag.status.pollinterval-ms";
public static final long TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT = 500;
@@ -1248,7 +1351,9 @@ public class TezConfiguration extends Configuration {
* Long value.
* Time to wait (in seconds) for apps to complete on MiniTezCluster shutdown.
*/
+ @Private
@ConfigurationScope(Scope.TEST)
+ @ConfigurationProperty(type="long")
public static final String TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS =
TEZ_PREFIX + "test.minicluster.app.wait.on.shutdown.secs";
public static final long TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS_DEFAULT = 30;
@@ -1260,6 +1365,7 @@ public class TezConfiguration extends Configuration {
*/
@Private
@ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty(type="long")
public static final String TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS =
TEZ_PREFIX + "client.diagnostics.wait.timeout-ms";
@Private
@@ -1273,6 +1379,7 @@ public class TezConfiguration extends Configuration {
*/
@Private
@ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty
public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS =
TEZ_PREFIX + "java.opts.checker.class";
@@ -1282,6 +1389,7 @@ public class TezConfiguration extends Configuration {
*/
@Private
@ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED =
TEZ_PREFIX + "java.opts.checker.enabled";
public static final boolean TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT = true;
@@ -1292,8 +1400,8 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty(type="long")
public static final String TEZ_CLIENT_HARD_KILL_TIMEOUT_MS = TEZ_PREFIX + "client.timeout-ms";
-
public static final long TEZ_CLIENT_HARD_KILL_TIMEOUT_MS_DEFAULT = 30 * 1000L;
/**
@@ -1302,7 +1410,7 @@ public class TezConfiguration extends Configuration {
* Expert level setting.
*/
@ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty(type="boolean")
public static final String TEZ_CLIENT_ASYNCHRONOUS_STOP = TEZ_PREFIX + "client.asynchronous-stop";
-
public static final boolean TEZ_CLIENT_ASYNCHRONOUS_STOP_DEFAULT = true;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml
index 9c7fa61..0605c63 100644
--- a/tez-dist/pom.xml
+++ b/tez-dist/pom.xml
@@ -30,6 +30,11 @@
<dependencies>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-dag</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-runtime-library/pom.xml
----------------------------------------------------------------------
diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml
index 239d62d..efb9c12 100644
--- a/tez-runtime-library/pom.xml
+++ b/tez-runtime-library/pom.xml
@@ -138,4 +138,36 @@
</plugin>
</plugins>
</build>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <reportSets>
+ <reportSet>
+ <id>configgen</id>
+ <configuration>
+ <doclet>org.apache.tez.tools.javadoc.doclet.ConfigStandardDoclet</doclet>
+ <docletArtifacts>
+ <docletArtifact>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-javadoc-tools</artifactId>
+ <version>${project.version}</version>
+ </docletArtifact>
+ </docletArtifacts>
+ <destDir>apidocs/configs</destDir>
+ <name>TezConfigGenerator</name>
+ <description>Tez Configuration documentation</description>
+ </configuration>
+ <reports>
+ <report>javadoc</report>
+ <report>aggregate</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ </plugins>
+ </reporting>
+
</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
index 5d4ed88..a84448f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.annotation.ConfigurationClass;
+import org.apache.tez.common.annotation.ConfigurationProperty;
import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl;
/**
@@ -37,11 +39,14 @@ import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.Sort
* <p/>
* Note for developers: Whenever a new key is added to this file, it must also be added to the set of
* known tezRuntimeKeys.
+ * @see <a href="../../../../../../configs/TezRuntimeConfiguration.html">Detailed Configuration Information</a>
+ * @see <a href="../../../../../configs/tez-runtime-default-template.xml">XML-based Config Template</a>
*/
// TODO EVENTUALLY A description for each property.
@Public
@Evolving
+@ConfigurationClass(templateFileName = "tez-runtime-default-template.xml")
public class TezRuntimeConfiguration {
private static final String TEZ_RUNTIME_PREFIX = "tez.runtime.";
@@ -69,6 +74,7 @@ public class TezRuntimeConfiguration {
/**
* Configuration key to enable/disable IFile readahead.
*/
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_IFILE_READAHEAD = TEZ_RUNTIME_PREFIX +
"ifile.readahead";
public static final boolean TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT = true;
@@ -76,6 +82,7 @@ public class TezRuntimeConfiguration {
/**
* Configuration key to set the IFile readahead length in bytes.
*/
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_IFILE_READAHEAD_BYTES = TEZ_RUNTIME_PREFIX +
"ifile.readahead.bytes";
public static final int TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT =
@@ -91,20 +98,24 @@ public class TezRuntimeConfiguration {
public static final String TEZ_RUNTIME_IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_IO_SORT_FACTOR = TEZ_RUNTIME_PREFIX +
"io.sort.factor";
public static final int TEZ_RUNTIME_IO_SORT_FACTOR_DEFAULT = 100;
+ @ConfigurationProperty(type = "float")
public static final String TEZ_RUNTIME_SORT_SPILL_PERCENT = TEZ_RUNTIME_PREFIX +
"sort.spill.percent";
public static final float TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT = 0.8f;
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_IO_SORT_MB = TEZ_RUNTIME_PREFIX + "io.sort.mb";
public static final int TEZ_RUNTIME_IO_SORT_MB_DEFAULT = 100;
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES = TEZ_RUNTIME_PREFIX +
"index.cache.memory.limit.bytes";
public static final int TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT =
@@ -112,6 +123,7 @@ public class TezRuntimeConfiguration {
// TODO Use the default value
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_COMBINE_MIN_SPILLS = TEZ_RUNTIME_PREFIX +
"combine.min.spills";
public static final int TEZ_RUNTIME_COMBINE_MIN_SPILLS_DEFAULT = 3;
@@ -125,10 +137,12 @@ public class TezRuntimeConfiguration {
* - PIPELINED ( default )
* {@link org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl}
*/
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_SORTER_CLASS = TEZ_RUNTIME_PREFIX +
"sorter.class";
public static final String TEZ_RUNTIME_SORTER_CLASS_DEFAULT = SorterImpl.PIPELINED.name();
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS = TEZ_RUNTIME_PREFIX +
"pipelined.sorter.sort.threads";
public static final int TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT = 2;
@@ -137,6 +151,7 @@ public class TezRuntimeConfiguration {
* Report partition statistics (e.g better scheduling in ShuffleVertexManager). TEZ-2496
* This can be enabled/disabled at vertex level.
*/
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_REPORT_PARTITION_STATS = TEZ_RUNTIME_PREFIX +
"report.partition.stats";
public static final boolean TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT = true;
@@ -144,6 +159,7 @@ public class TezRuntimeConfiguration {
/**
* Size of the buffer to use if not writing directly to disk.
*/
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB = TEZ_RUNTIME_PREFIX +
"unordered.output.buffer.size-mb";
public static final int TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB_DEFAULT = 100;
@@ -153,6 +169,7 @@ public class TezRuntimeConfiguration {
* This is only meant to be used by unit tests for now.
*/
@Private
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES =
TEZ_RUNTIME_PREFIX +
"unordered.output.max-per-buffer.size-bytes";
@@ -161,125 +178,140 @@ public class TezRuntimeConfiguration {
* Specifies a partitioner class, which is used in Tez Runtime components
* like OnFileSortedOutput
*/
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_PARTITIONER_CLASS =
TEZ_RUNTIME_PREFIX + "partitioner.class";
/**
* Specifies a combiner class (primarily for Shuffle)
*/
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_COMBINER_CLASS = TEZ_RUNTIME_PREFIX + "combiner.class";
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES = TEZ_RUNTIME_PREFIX +
"shuffle.parallel.copies";
public static final int TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT = 20;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT = TEZ_RUNTIME_PREFIX +
"shuffle.fetch.failures.limit";
public static final int TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT_DEFAULT = 5;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE =
TEZ_RUNTIME_PREFIX +
"shuffle.fetch.max.task.output.at.once";
public final static int TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE_DEFAULT
= 20;
-
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR = TEZ_RUNTIME_PREFIX +
"shuffle.notify.readerror";
public static final boolean TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR_DEFAULT = true;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT = TEZ_RUNTIME_PREFIX +
"shuffle.connect.timeout";
public static final int TEZ_RUNTIME_SHUFFLE_STALLED_COPY_TIMEOUT_DEFAULT =
3 * 60 * 1000;
-
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED = TEZ_RUNTIME_PREFIX +
"shuffle.keep-alive.enabled";
public static final boolean TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED_DEFAULT = false;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS = TEZ_RUNTIME_PREFIX +
"shuffle.keep-alive.max.connections";
public static final int TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS_DEFAULT = 20;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT =
TEZ_RUNTIME_PREFIX + "shuffle.read.timeout";
public final static int TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT_DEFAULT =
3 * 60 * 1000;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE =
TEZ_RUNTIME_PREFIX + "shuffle.buffersize";
public final static int TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE_DEFAULT =
8 * 1024;
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_SHUFFLE_USE_ASYNC_HTTP = TEZ_RUNTIME_PREFIX +
"shuffle.use.async.http";
public static final boolean TEZ_RUNTIME_SHUFFLE_USE_ASYNC_HTTP_DEFAULT = false;
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_SSL = TEZ_RUNTIME_PREFIX +
"shuffle.ssl.enable";
public static final boolean TEZ_RUNTIME_SHUFFLE_ENABLE_SSL_DEFAULT = false;
-
+ @ConfigurationProperty(type = "float")
public static final String TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT = TEZ_RUNTIME_PREFIX +
"shuffle.fetch.buffer.percent";
public static final float TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT_DEFAULT =
0.90f;
-
+ @ConfigurationProperty(type = "float")
public static final String TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT = TEZ_RUNTIME_PREFIX +
"shuffle.memory.limit.percent";
public static final float TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT_DEFAULT =
0.25f;
// Rename to fraction
+ @ConfigurationProperty(type = "float")
public static final String TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT = TEZ_RUNTIME_PREFIX +
"shuffle.merge.percent";
public static final float TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT_DEFAULT = 0.90f;
-
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS = TEZ_RUNTIME_PREFIX +
"shuffle.memory-to-memory.segments";
-
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM = TEZ_RUNTIME_PREFIX +
"shuffle.memory-to-memory.enable";
public static final boolean TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM_DEFAULT =
false;
-
+ @ConfigurationProperty(type = "float")
public static final String TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT = TEZ_RUNTIME_PREFIX +
"task.input.post-merge.buffer.percent";
public static final float TEZ_RUNTIME_INPUT_BUFFER_PERCENT_DEFAULT = 0.0f;
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_GROUP_COMPARATOR_CLASS = TEZ_RUNTIME_PREFIX +
"group.comparator.class";
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_INTERNAL_SORTER_CLASS = TEZ_RUNTIME_PREFIX +
"internal.sorter.class";
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_KEY_COMPARATOR_CLASS =
TEZ_RUNTIME_PREFIX + "key.comparator.class";
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_KEY_CLASS = TEZ_RUNTIME_PREFIX + "key.class";
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_VALUE_CLASS = TEZ_RUNTIME_PREFIX + "value.class";
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_COMPRESS = TEZ_RUNTIME_PREFIX + "compress";
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_COMPRESS_CODEC = TEZ_RUNTIME_PREFIX + "compress.codec";
// TODO Move this key to MapReduce
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS =
TEZ_RUNTIME_PREFIX + "key.secondary.comparator.class";
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED =
TEZ_RUNTIME_PREFIX +
"empty.partitions.info-via-events.enabled";
@@ -288,6 +320,7 @@ public class TezRuntimeConfiguration {
/**
* If the shuffle input is on the local host bypass the http fetch and access the files directly
*/
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH = TEZ_RUNTIME_PREFIX + "optimize.local.fetch";
public static final boolean TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT = true;
@@ -298,6 +331,7 @@ public class TezRuntimeConfiguration {
* Ensure to set tez.runtime.enable.final-merge.in.output=false.
* Speculative execution needs to be turned off when using this parameter. //TODO: TEZ-2132
*/
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED =
TEZ_RUNTIME_PREFIX + "pipelined-shuffle.enabled";
public static final boolean TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED_DEFAULT = false;
@@ -306,6 +340,7 @@ public class TezRuntimeConfiguration {
* Expert level setting. Enable final merge in ordered (defaultsorter/pipelinedsorter) outputs.
* Speculative execution needs to be turned off when disabling this parameter. //TODO: TEZ-2132
*/
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT =
TEZ_RUNTIME_PREFIX + "enable.final-merge.in.output";
public static final boolean TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT_DEFAULT = true;
@@ -314,6 +349,7 @@ public class TezRuntimeConfiguration {
/**
* Share data fetched between tasks running on the same host if applicable
*/
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH = TEZ_RUNTIME_PREFIX
+ "optimize.shared.fetch";
@@ -328,6 +364,7 @@ public class TezRuntimeConfiguration {
*/
@Unstable
@Private
+ @ConfigurationProperty(type = "boolean")
public static final String TEZ_RUNTIME_CLEANUP_FILES_ON_INTERRUPT = TEZ_RUNTIME_PREFIX
+ "cleanup.files.on.interrupt";
public static final boolean TEZ_RUNTIME_CLEANUP_FILES_ON_INTERRUPT_DEFAULT = false;
@@ -339,12 +376,14 @@ public class TezRuntimeConfiguration {
* Value: Boolean
* Whether to publish configuration information to History logger. Default false.
*/
+ @ConfigurationProperty
public static final String TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT =
TEZ_RUNTIME_PREFIX + "convert.user-payload.to.history-text";
public static final boolean TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT = false;
@Unstable
@Private
+ @ConfigurationProperty(type = "integer")
public static final String TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS = TEZ_RUNTIME_PREFIX +
"merge.progress.records";
public static final long TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT = 10000;
@@ -448,4 +487,4 @@ public class TezRuntimeConfiguration {
public static Map<String, String> getOtherConfigDefaults() {
return Collections.unmodifiableMap(otherConfMap);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/pom.xml
----------------------------------------------------------------------
diff --git a/tez-tools/pom.xml b/tez-tools/pom.xml
index af1d304..bf3053e 100644
--- a/tez-tools/pom.xml
+++ b/tez-tools/pom.xml
@@ -28,7 +28,9 @@
<modules>
<module>analyzers</module>
+ <module>tez-javadoc-tools</module>
</modules>
+
<build>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-tools/tez-javadoc-tools/findbugs-exclude.xml b/tez-tools/tez-javadoc-tools/findbugs-exclude.xml
new file mode 100644
index 0000000..ad8f6e4
--- /dev/null
+++ b/tez-tools/tez-javadoc-tools/findbugs-exclude.xml
@@ -0,0 +1,22 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+ <Match>
+ <Class name="org.apache.tez.tools.javadoc.model.ConfigProperty"/>
+ <Field name="validValues"/>
+ <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/pom.xml
----------------------------------------------------------------------
diff --git a/tez-tools/tez-javadoc-tools/pom.xml b/tez-tools/tez-javadoc-tools/pom.xml
new file mode 100644
index 0000000..11290cc
--- /dev/null
+++ b/tez-tools/tez-javadoc-tools/pom.xml
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-tools</artifactId>
+ <version>0.8.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>tez-javadoc-tools</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>platform-macosx</id>
+ <activation>
+ <file>
+ <exists>${java.home}/../Classes/classes.jar</exists>
+ </file>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.sun</groupId>
+ <artifactId>tools</artifactId>
+ <version>${java.version}</version>
+ <scope>system</scope>
+ <systemPath>${java.home}/../Classes/classes.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>other-os</id>
+ <activation>
+ <file>
+ <exists>${java.home}/../lib/tools.jar</exists>
+ </file>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.sun</groupId>
+ <artifactId>tools</artifactId>
+ <version>${java.version}</version>
+ <scope>system</scope>
+ <systemPath>${java.home}/../lib/tools.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java
----------------------------------------------------------------------
diff --git a/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java
new file mode 100644
index 0000000..6cc9ced
--- /dev/null
+++ b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tools.javadoc.doclet;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.tez.common.annotation.ConfigurationClass;
+import org.apache.tez.common.annotation.ConfigurationProperty;
+import org.apache.tez.tools.javadoc.model.Config;
+import org.apache.tez.tools.javadoc.model.ConfigProperty;
+import org.apache.tez.tools.javadoc.util.HtmlWriter;
+import org.apache.tez.tools.javadoc.util.XmlWriter;
+
+import com.sun.javadoc.AnnotationDesc;
+import com.sun.javadoc.AnnotationDesc.ElementValuePair;
+import com.sun.javadoc.ClassDoc;
+import com.sun.javadoc.DocErrorReporter;
+import com.sun.javadoc.FieldDoc;
+import com.sun.javadoc.LanguageVersion;
+import com.sun.javadoc.RootDoc;
+import com.sun.tools.doclets.standard.Standard;
+
+public class ConfigStandardDoclet {
+
+ private static final String DEBUG_SWITCH = "-debug";
+ private static boolean debugMode = false;
+
+ public static LanguageVersion languageVersion() {
+ return LanguageVersion.JAVA_1_5;
+ }
+
+ private static void logMessage(String message) {
+ if (!debugMode) {
+ return;
+ }
+ System.out.println(message);
+ }
+
+ public static boolean start(RootDoc root) {
+ //look for debug flag
+ for (String[] opts : root.options()) {
+ for (String opt : opts) {
+ if (opt.equals(DEBUG_SWITCH)) {
+ debugMode = true;
+ }
+ }
+ }
+
+ logMessage("Running doclet " + ConfigStandardDoclet.class.getSimpleName());
+ ClassDoc[] classes = root.classes();
+ for (int i = 0; i < classes.length; ++i) {
+ processDoc(classes[i]);
+ }
+
+ return true;
+ }
+
+ private static void processDoc(ClassDoc doc) {
+ logMessage("Parsing : " + doc);
+ if (!doc.isClass()) {
+ logMessage("Ignoring non-class: " + doc);
+ return;
+ }
+
+ AnnotationDesc[] annotations = doc.annotations();
+ boolean isConfigClass = false;
+ String templateName = null;
+ for (AnnotationDesc annotation : annotations) {
+ logMessage("Checking annotation: " + annotation.annotationType());
+ if (annotation.annotationType().qualifiedTypeName().equals(
+ ConfigurationClass.class.getName())) {
+ isConfigClass = true;
+ for (ElementValuePair element : annotation.elementValues()) {
+ if (element.element().name().equals("templateFileName")) {
+ templateName = stripQuotes(element.value().toString());
+ }
+ }
+ break;
+ }
+ }
+
+ if (!isConfigClass) {
+ logMessage("Ignoring non-config class: " + doc);
+ return;
+ }
+
+ logMessage("Processing config class: " + doc);
+ Config config = new Config(doc.name(), templateName);
+ Map<String, ConfigProperty> configProperties = config.configProperties;
+
+ FieldDoc[] fields = doc.fields();
+ for (FieldDoc field : fields) {
+ if (field.isPrivate()) {
+ logMessage("Skipping private field: " + field);
+ continue;
+ }
+ if (!field.isStatic()) {
+ logMessage("Skipping non-static field: " + field);
+ continue;
+ }
+
+ if (field.name().endsWith("_PREFIX")) {
+ logMessage("Skipping non-config prefix constant field: " + field);
+ continue;
+ }
+ if (field.name().equals("TEZ_SITE_XML")) {
+ logMessage("Skipping constant field: " + field);
+ continue;
+ }
+
+ if (field.name().endsWith("_DEFAULT")) {
+
+ String name = field.name().substring(0,
+ field.name().lastIndexOf("_DEFAULT"));
+ if (!configProperties.containsKey(name)) {
+ configProperties.put(name, new ConfigProperty());
+ }
+ ConfigProperty configProperty = configProperties.get(name);
+ if (field.constantValue() == null) {
+ logMessage("Got null constant value"
+ + ", name=" + name
+ + ", field=" + field.name()
+ + ", val=" + field.constantValueExpression());
+ configProperty.defaultValue = field.constantValueExpression();
+ } else {
+ configProperty.defaultValue = field.constantValue().toString();
+ }
+ configProperty.inferredType = field.type().simpleTypeName();
+
+ if (name.equals("TEZ_AM_STAGING_DIR") && configProperty.defaultValue != null) {
+ String defaultValue = configProperty.defaultValue;
+ defaultValue = defaultValue.replace(System.getProperty("user.name"), "${user.name}");
+ configProperty.defaultValue = defaultValue;
+ }
+
+ continue;
+ }
+
+ String name = field.name();
+ if (!configProperties.containsKey(name)) {
+ configProperties.put(name, new ConfigProperty());
+ }
+ ConfigProperty configProperty = configProperties.get(name);
+ configProperty.propertyName = field.constantValue().toString();
+
+ AnnotationDesc[] annotationDescs = field.annotations();
+
+ for (AnnotationDesc annotationDesc : annotationDescs) {
+
+ if (annotationDesc.annotationType().qualifiedTypeName().equals(
+ Private.class.getCanonicalName())) {
+ configProperty.isPrivate = true;
+ }
+ if (annotationDesc.annotationType().qualifiedTypeName().equals(
+ Unstable.class.getCanonicalName())) {
+ configProperty.isUnstable = true;
+ }
+ if (annotationDesc.annotationType().qualifiedTypeName().equals(
+ Evolving.class.getCanonicalName())) {
+ configProperty.isEvolving = true;
+ }
+ if (annotationDesc.annotationType().qualifiedTypeName().equals(
+ ConfigurationProperty.class.getCanonicalName())) {
+ configProperty.isValidConfigProp = true;
+
+ boolean foundType = false;
+ for (ElementValuePair element : annotationDesc.elementValues()) {
+ if (element.element().name().equals("type")) {
+ configProperty.type = stripQuotes(element.value().toString());
+ foundType = true;
+ } else {
+ logMessage("Unhandled annotation property: " + element.element().name());
+ }
+ }
+ }
+ }
+
+ configProperty.description = field.commentText();
+
+ }
+
+ HtmlWriter writer = new HtmlWriter();
+ try {
+ writer.write(config);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ XmlWriter xmlWriter = new XmlWriter();
+ try {
+ xmlWriter.write(config);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private static String stripQuotes(String s) {
+ if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') {
+ return s.substring(1, s.length()-1);
+ }
+ return s;
+ }
+
+ public static int optionLength(String option) {
+ return Standard.optionLength(option);
+ }
+
+ public static boolean validOptions(String options[][], DocErrorReporter reporter) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java
----------------------------------------------------------------------
diff --git a/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java
new file mode 100644
index 0000000..604d48a
--- /dev/null
+++ b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/Config.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tools.javadoc.model;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+public class Config {
+
+ public final String templateName;
+ public final String configName;
+ public Map<String, ConfigProperty> configProperties;
+
+ public Config(String configName, String templateName) {
+ this.configName = configName;
+ this.templateName = templateName;
+ this.configProperties = new TreeMap<String, ConfigProperty>();
+ }
+
+ public Config() {
+ this(null, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/50df8651/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java
----------------------------------------------------------------------
diff --git a/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java
new file mode 100644
index 0000000..94c7e27
--- /dev/null
+++ b/tez-tools/tez-javadoc-tools/src/main/java/org/apache/tez/tools/javadoc/model/ConfigProperty.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tools.javadoc.model;
+
+public class ConfigProperty {
+
+ public String propertyName;
+ public String defaultValue;
+ public String description;
+ public String type = "string";
+ public boolean isPrivate = false;
+ public boolean isUnstable = false;
+ public boolean isEvolving = false;
+ public boolean isValidConfigProp = false;
+ public String[] validValues;
+ public String inferredType;
+
+ @Override
+ public String toString() {
+ return "name=" + propertyName
+ + ", defaultValue=" + defaultValue
+ + ", description=" + description
+ + ", type=" + type
+ + ", private=" + isPrivate
+ + ", validValues=" + (validValues == null ? "null" : validValues)
+ + ", isConfigProp=" + isValidConfigProp;
+ }
+}
+
+