You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2016/10/18 23:45:30 UTC
[44/50] [abbrv] hadoop git commit: YARN-5718. TimelineClient (and
other places in YARN) shouldn't over-write HDFS client retry settings which
could cause unexpected behavior. Contributed by Junping Du.
YARN-5718. TimelineClient (and other places in YARN) shouldn't over-write HDFS client retry settings which could cause unexpected behavior. Contributed by Junping Du.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b733a6f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b733a6f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b733a6f8
Branch: refs/heads/HADOOP-13070
Commit: b733a6f86262522e535cebc972baecbe6a6eab50
Parents: b154d3e
Author: Xuan <xg...@apache.org>
Authored: Tue Oct 18 11:04:49 2016 -0700
Committer: Xuan <xg...@apache.org>
Committed: Tue Oct 18 11:06:47 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 17 +----------------
.../api/impl/FileSystemTimelineWriter.java | 7 -------
.../nodelabels/FileSystemNodeLabelsStore.java | 7 +------
.../src/main/resources/yarn-default.xml | 20 --------------------
.../recovery/FileSystemRMStateStore.java | 5 -----
.../recovery/TestFSRMStateStore.java | 4 ----
6 files changed, 2 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 3bd0dcc..1a30c32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -695,10 +695,6 @@ public class YarnConfiguration extends Configuration {
/** URI for FileSystemRMStateStore */
public static final String FS_RM_STATE_STORE_URI = RM_PREFIX
+ "fs.state-store.uri";
- public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX
- + "fs.state-store.retry-policy-spec";
- public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC =
- "2000, 500";
public static final String FS_RM_STATE_STORE_NUM_RETRIES =
RM_PREFIX + "fs.state-store.num-retries";
@@ -1974,13 +1970,6 @@ public class YarnConfiguration extends Configuration {
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT
= 24 * 60 * 60;
- public static final String
- TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC =
- TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "retry-policy-spec";
- public static final String
- DEFAULT_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC =
- "2000, 500";
-
public static final String TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE =
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX
+ "leveldb-cache-read-cache-size";
@@ -2600,11 +2589,7 @@ public class YarnConfiguration extends Configuration {
/** URI for NodeLabelManager */
public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
+ "fs-store.root-dir";
- public static final String FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
- NODE_LABELS_PREFIX + "fs-store.retry-policy-spec";
- public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
- "2000, 500";
-
+
/**
* Flag to indicate if the node labels feature enabled, by default it's
* disabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 55d6bd2..b1284e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -106,13 +106,6 @@ public class FileSystemTimelineWriter extends TimelineWriter{
super(authUgi, client, resURI);
Configuration fsConf = new Configuration(conf);
- fsConf.setBoolean("dfs.client.retry.policy.enabled", true);
- String retryPolicy =
- fsConf.get(YarnConfiguration.
- TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC,
- YarnConfiguration.
- DEFAULT_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC);
- fsConf.set("dfs.client.retry.policy.spec", retryPolicy);
activePath = new Path(fsConf.get(
YarnConfiguration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
index a65349b..9a2b8be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
@@ -95,13 +95,8 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
void setFileSystem(Configuration conf) throws IOException {
Configuration confCopy = new Configuration(conf);
- confCopy.setBoolean("dfs.client.retry.policy.enabled", true);
- String retryPolicy =
- confCopy.get(YarnConfiguration.FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC,
- YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC);
- confCopy.set("dfs.client.retry.policy.spec", retryPolicy);
fs = fsWorkingPath.getFileSystem(confCopy);
-
+
// if it's local file system, use RawLocalFileSystem instead of
// LocalFileSystem, the latter one doesn't support append.
if (fs.getScheme().equals("file")) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f37c689..72e026d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -522,16 +522,6 @@
</property>
<property>
- <description>hdfs client retry policy specification. hdfs client retry
- is always enabled. Specified in pairs of sleep-time and number-of-retries
- and (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on
- average, the following n1 retries sleep t1 milliseconds on average, and so on.
- </description>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
<description>the number of retries to recover from IOException in
FileSystemRMStateStore.
</description>
@@ -2483,16 +2473,6 @@
<property>
<description>
- Retry policy used for FileSystem node label store. The policy is
- specified by N pairs of sleep-time in milliseconds and number-of-retries
- "s1,n1,s2,n2,...".
- </description>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <description>
URI for NodeLabelManager. The default value is
/tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index a228037..b9a4374 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -142,11 +142,6 @@ public class FileSystemRMStateStore extends RMStateStore {
// authenticated with kerberos so we are good to create a file-system
// handle.
fsConf = new Configuration(getConfig());
- fsConf.setBoolean("dfs.client.retry.policy.enabled", true);
- String retryPolicy =
- fsConf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
- YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC);
- fsConf.set("dfs.client.retry.policy.spec", retryPolicy);
String scheme = fsWorkingPath.toUri().getScheme();
if (scheme == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 61088e1..6f0d53f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -113,8 +113,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
YarnConfiguration conf = new YarnConfiguration();
conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
workingDirPathURI.toString());
- conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
- "100,6000");
conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
900L);
@@ -325,8 +323,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
YarnConfiguration conf = new YarnConfiguration();
conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
workingDirPathURI.toString());
- conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
- "100,6000");
this.store = new TestFileSystemRMStore(conf) {
Version storedVersion = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org